def test_object_upload_iterator(self):
# target_object是想要列举的文件，而intact_object则不是。
# 这里intact_object故意以target_object为前缀
target_object = self.random_key()
intact_object = self.random_key()
target_list = []
intact_list = []
# 准备分片
for i in range(10):
target_list.append(self.bucket.init_multipart_upload(target_object).upload_id)
intact_list.append(self.bucket.init_multipart_upload(intact_object).upload_id)
# 验证：max_uploads能被分片数整除
uploads_got = []
for u in oss2.ObjectUploadIterator(self.bucket, target_object, max_uploads=5):
uploads_got.append(u.upload_id)
self.assertEqual(sorted(target_list), uploads_got)
# 验证：max_uploads不能被分片数整除
uploads_got = []
for u in oss2.ObjectUploadIterator(self.bucket, target_object, max_uploads=3):
uploads_got.append(u.upload_id)
self.assertEqual(sorted(target_list), uploads_got)
# 清理
for upload_id in target_list:
self.bucket.abort_multipart_upload(target_object, upload_id)
for upload_id in intact_list:
self.bucket.abort_multipart_upload(intact_object, upload_id)
def test_object_upload_iterator(self):
key = 'requestpayment-test-object-upload-iterator'
content = 'test-content'
self.bucket.put_object(key, content)
# ObjectUploadIterator without payer setting, should be failed.
access_err_flag = False
try:
up_iter = oss2.ObjectUploadIterator(self.payer_bucket, key)
for up in up_iter:
pass
except oss2.exceptions.AccessDenied:
access_err_flag = True
self.assertEqual(access_err_flag, True)
# ObjectUploadIterator with payer setting, should be successful.
headers = dict()
headers[OSS_REQUEST_PAYER] = "requester"
up_iter = oss2.ObjectUploadIterator(self.payer_bucket,
key,
headers=headers)
for up in up_iter:
pass
self.bucket.delete_object(key)
def __test_resume(self, content_size, uploaded_parts, expected_unfinished=0):
part_size = 100 * 1024
num_parts = (content_size + part_size - 1) // part_size
key = 'resume-' + random_string(32)
content = random_bytes(content_size)
pathname = self._prepare_temp_file(content)
upload_id = self.bucket.init_multipart_upload(key).upload_id
for part_number in uploaded_parts:
start = (part_number -1) * part_size
if part_number == num_parts:
end = content_size
else:
end = start + part_size
self.bucket.upload_part(key, upload_id, part_number, content[start:end])
oss2.resumable._rebuild_record(pathname, oss2.resumable.make_upload_store(), self.bucket, key, upload_id, part_size)
oss2.resumable_upload(self.bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024)
result = self.bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(len(list(oss2.ObjectUploadIterator(self.bucket, key))), expected_unfinished)
self.bucket.delete_object(key)
def __test_interrupt(self, content_size, failed_part_number,
expected_unfinished=0,
modify_record_func=None):
orig_upload_part = oss2.Bucket.upload_part
def upload_part(self, key, upload_id, part_number, data):
if part_number == failed_part_number:
raise RuntimeError
else:
return orig_upload_part(self, key, upload_id, part_number, data)
key = 'resume-' + random_string(32)
content = random_bytes(content_size)
pathname = self._prepare_temp_file(content)
with patch.object(oss2.Bucket, 'upload_part', side_effect=upload_part, autospec=True) as mock_upload_part:
self.assertRaises(RuntimeError, oss2.resumable_upload, self.bucket, key, pathname,
multipart_threshold=0,
part_size=100 * 1024)
if modify_record_func:
modify_record_func(oss2.resumable.make_upload_store(), self.bucket.bucket_name, key, pathname)
oss2.resumable_upload(self.bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024)
self.assertEqual(len(list(oss2.ObjectUploadIterator(self.bucket, key))), expected_unfinished)
def __test_resume(self,
content_size,
uploaded_parts,
expected_unfinished=0):
bucket = random.choice(
[self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
part_size = 100 * 1024
num_parts = (content_size + part_size - 1) // part_size
key = 'resume-' + random_string(32)
content = random_bytes(content_size)
encryption_flag = isinstance(bucket, oss2.CryptoBucket)
context = None
pathname = self._prepare_temp_file(content)
if encryption_flag:
context = models.MultipartUploadCryptoContext(
content_size, part_size)
upload_id = bucket.init_multipart_upload(
key, upload_context=context).upload_id
else:
upload_id = bucket.init_multipart_upload(key).upload_id
for part_number in uploaded_parts:
start = (part_number - 1) * part_size
if part_number == num_parts:
end = content_size
else:
end = start + part_size
if encryption_flag:
bucket.upload_part(key,
upload_id,
part_number,
content[start:end],
upload_context=context)
else:
bucket.upload_part(key, upload_id, part_number,
content[start:end])
self._rebuild_record(pathname, oss2.resumable.make_upload_store(),
bucket, key, upload_id, part_size, context)
oss2.resumable_upload(bucket,
key,
pathname,
multipart_threshold=0,
part_size=100 * 1024)
result = bucket.get_object(key)
self.assertEqual(content, result.read())
self.assertEqual(
len(list(oss2.ObjectUploadIterator(self.bucket, key))),
expected_unfinished)
bucket.delete_object(key)
def __test_interrupt(self, content_size, failed_part_number, expected_unfinished=0, modify_record_func=None):
bucket = random.choice([self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket])
encryption_flag = isinstance(bucket, oss2.CryptoBucket)
orig_upload_part = bucket.upload_part
if encryption_flag:
def upload_part(self, key, upload_id, part_number, data, progress_callback=None, headers=None,
upload_context=None):
if part_number == failed_part_number:
raise RuntimeError
else:
return orig_upload_part(key, upload_id, part_number, data, progress_callback, headers,
upload_context)
else:
def upload_part(self, key, upload_id, part_number, data, progress_callback=None, headers=None):
if part_number == failed_part_number:
raise RuntimeError
else:
return orig_upload_part(key, upload_id, part_number, data, progress_callback, headers)
key = 'ResumableUpload-' + random_string(32)
content = random_bytes(content_size)
pathname = self._prepare_temp_file(content)
if encryption_flag:
with patch.object(oss2.CryptoBucket, 'upload_part', side_effect=upload_part,
autospec=True) as mock_upload_part:
self.assertRaises(RuntimeError, oss2.resumable_upload, bucket, key, pathname, multipart_threshold=0,
part_size=100 * 1024)
else:
with patch.object(oss2.Bucket, 'upload_part', side_effect=upload_part, autospec=True) as mock_upload_part:
self.assertRaises(RuntimeError, oss2.resumable_upload, bucket, key, pathname, multipart_threshold=0,
part_size=100 * 1024)
if modify_record_func:
modify_record_func(oss2.resumable.make_upload_store(), bucket.bucket_name, key, pathname)
oss2.resumable_upload(bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024)
self.assertEqual(len(list(oss2.ObjectUploadIterator(self.bucket, key))), expected_unfinished)
