def testUploadFile_FailFourTimes(self):
# max_retrials=3 as set in _UploadFile means that there can be up to 4
# attempts
self.copy_file_mock.side_effect = MakeRepeatMock(5)
tasks = self._MakeTestTasks(self._DEFAULT_NUM_TASKS)
with self.assertRaises(parallel.MultiError):
storage_parallel.UploadFiles(tasks)
def testUploadFile_OneFile(self):
tasks = self._MakeTestTasks(1)
storage_parallel.UploadFiles(tasks)
self.copy_file_mock.assert_called_once_with(
'local0',
storage_util.ObjectReference(self._TEST_BUCKET, 'remote0'))
self.get_pool_mock.assert_called_once_with(16)
def _RunTest(self, num_threads):
with storage_e2e_util.CloudStorageBucket(self.storage_client,
self.bucket_name,
self.Project()) as bucket_ref:
file_upload_tasks = self._MakeFileUploadTasks(bucket_ref)
storage_parallel.UploadFiles(file_upload_tasks, num_threads)
object_names = [f.dest_obj_ref.object for f in file_upload_tasks]
self._AssertFilesUploaded(bucket_ref, object_names)
def testProgressBar_NoTasks(self):
storage_parallel.UploadFiles([], num_threads=1, show_progress_bar=True)
self.AssertErrEquals(
'#============================================================#\n'
'#= Uploading 0 files to Google Cloud Storage =#\n'
'#============================================================#\n'
)
self.copy_file_mock.assert_not_called()
def _RunTestWithGivenParallelism(self, num_threads):
tasks = self._MakeTestTasks(self._DEFAULT_NUM_TASKS)
storage_parallel.UploadFiles(tasks, num_threads=num_threads)
for n in range(self._DEFAULT_NUM_TASKS):
self.copy_file_mock.assert_any_call(
'local{0}'.format(n),
storage_util.ObjectReference(self._TEST_BUCKET,
'remote{0}'.format(n)))
self.assertEqual(self.copy_file_mock.call_count, self._DEFAULT_NUM_TASKS)
self.get_pool_mock.assert_called_once_with(num_threads)
def _RunTestWithSuccessAfterNumTries(self, num_tries):
self.copy_file_mock.side_effect = MakeRepeatMock(num_tries)
tasks = self._MakeTestTasks(self._DEFAULT_NUM_TASKS)
storage_parallel.UploadFiles(tasks)
calls = []
self.assertEqual(self.copy_file_mock.call_count,
self._DEFAULT_NUM_TASKS * num_tries)
for n in range(self._DEFAULT_NUM_TASKS):
for _ in range(num_tries):
calls.append(mock.call(
'local{0}'.format(n),
storage_util.ObjectReference(self._TEST_BUCKET,
'remote{0}'.format(n))))
self.copy_file_mock.assert_has_calls(calls, any_order=True)
self.get_pool_mock.assert_called_once_with(16)
def testProgressBar_OneTask(self):
tasks = self._MakeTestTasks(1)
self.progress_bar_states = [
('#============================================================#\n'
'#= Uploading 1 file to Google Cloud Storage =#\n'
'#'),
]
storage_parallel.UploadFiles(tasks, num_threads=1, show_progress_bar=True)
self.assertEqual(self.copy_file_mock.call_count, 1)
self.AssertErrEquals(
'#============================================================#\n'
'#= Uploading 1 file to Google Cloud Storage =#\n'
'#============================================================#\n'
)
def _UploadFilesThreads(files_to_upload, bucket_ref):
"""Uploads files to App Engine Cloud Storage bucket using threads.
Args:
files_to_upload: dict {str: str}, map of checksum to local path
bucket_ref: storage_api.BucketReference, the reference to the bucket files
will be placed in.
Raises:
MultiError: if one or more errors occurred during file upload.
"""
threads_per_proc = (properties.VALUES.app.num_file_upload_threads.GetInt()
or storage_parallel.DEFAULT_NUM_THREADS)
tasks = []
# Have to sort files because the test framework requires a known order for
# mocked API calls.
for sha1_hash, path in sorted(files_to_upload.iteritems()):
task = storage_parallel.FileUploadTask(path, bucket_ref.ToBucketUrl(),
sha1_hash)
tasks.append(task)
storage_parallel.UploadFiles(tasks, threads_per_process=threads_per_proc)
def testUploadFile_NoFiles(self): storage_parallel.UploadFiles([]) self.copy_file_mock.assert_not_called() self.get_pool_mock.assert_called_once_with(16)
