def test_object_upload_iterator(self): # target_object是想要列举的文件,而intact_object则不是。 # 这里intact_object故意以target_object为前缀 target_object = self.random_key() intact_object = self.random_key() target_list = [] intact_list = [] # 准备分片 for i in range(10): target_list.append(self.bucket.init_multipart_upload(target_object).upload_id) intact_list.append(self.bucket.init_multipart_upload(intact_object).upload_id) # 验证:max_uploads能被分片数整除 uploads_got = [] for u in oss2.ObjectUploadIterator(self.bucket, target_object, max_uploads=5): uploads_got.append(u.upload_id) self.assertEqual(sorted(target_list), uploads_got) # 验证:max_uploads不能被分片数整除 uploads_got = [] for u in oss2.ObjectUploadIterator(self.bucket, target_object, max_uploads=3): uploads_got.append(u.upload_id) self.assertEqual(sorted(target_list), uploads_got) # 清理 for upload_id in target_list: self.bucket.abort_multipart_upload(target_object, upload_id) for upload_id in intact_list: self.bucket.abort_multipart_upload(intact_object, upload_id)
def test_object_upload_iterator(self): key = 'requestpayment-test-object-upload-iterator' content = 'test-content' self.bucket.put_object(key, content) # ObjectUploadIterator without payer setting, should be failed. access_err_flag = False try: up_iter = oss2.ObjectUploadIterator(self.payer_bucket, key) for up in up_iter: pass except oss2.exceptions.AccessDenied: access_err_flag = True self.assertEqual(access_err_flag, True) # ObjectUploadIterator with payer setting, should be successful. headers = dict() headers[OSS_REQUEST_PAYER] = "requester" up_iter = oss2.ObjectUploadIterator(self.payer_bucket, key, headers=headers) for up in up_iter: pass self.bucket.delete_object(key)
def __test_resume(self, content_size, uploaded_parts, expected_unfinished=0): part_size = 100 * 1024 num_parts = (content_size + part_size - 1) // part_size key = 'resume-' + random_string(32) content = random_bytes(content_size) pathname = self._prepare_temp_file(content) upload_id = self.bucket.init_multipart_upload(key).upload_id for part_number in uploaded_parts: start = (part_number -1) * part_size if part_number == num_parts: end = content_size else: end = start + part_size self.bucket.upload_part(key, upload_id, part_number, content[start:end]) oss2.resumable._rebuild_record(pathname, oss2.resumable.make_upload_store(), self.bucket, key, upload_id, part_size) oss2.resumable_upload(self.bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024) result = self.bucket.get_object(key) self.assertEqual(content, result.read()) self.assertEqual(len(list(oss2.ObjectUploadIterator(self.bucket, key))), expected_unfinished) self.bucket.delete_object(key)
def __test_interrupt(self, content_size, failed_part_number, expected_unfinished=0, modify_record_func=None): orig_upload_part = oss2.Bucket.upload_part def upload_part(self, key, upload_id, part_number, data): if part_number == failed_part_number: raise RuntimeError else: return orig_upload_part(self, key, upload_id, part_number, data) key = 'resume-' + random_string(32) content = random_bytes(content_size) pathname = self._prepare_temp_file(content) with patch.object(oss2.Bucket, 'upload_part', side_effect=upload_part, autospec=True) as mock_upload_part: self.assertRaises(RuntimeError, oss2.resumable_upload, self.bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024) if modify_record_func: modify_record_func(oss2.resumable.make_upload_store(), self.bucket.bucket_name, key, pathname) oss2.resumable_upload(self.bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024) self.assertEqual(len(list(oss2.ObjectUploadIterator(self.bucket, key))), expected_unfinished)
def __test_resume(self, content_size, uploaded_parts, expected_unfinished=0): bucket = random.choice( [self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket]) part_size = 100 * 1024 num_parts = (content_size + part_size - 1) // part_size key = 'resume-' + random_string(32) content = random_bytes(content_size) encryption_flag = isinstance(bucket, oss2.CryptoBucket) context = None pathname = self._prepare_temp_file(content) if encryption_flag: context = models.MultipartUploadCryptoContext( content_size, part_size) upload_id = bucket.init_multipart_upload( key, upload_context=context).upload_id else: upload_id = bucket.init_multipart_upload(key).upload_id for part_number in uploaded_parts: start = (part_number - 1) * part_size if part_number == num_parts: end = content_size else: end = start + part_size if encryption_flag: bucket.upload_part(key, upload_id, part_number, content[start:end], upload_context=context) else: bucket.upload_part(key, upload_id, part_number, content[start:end]) self._rebuild_record(pathname, oss2.resumable.make_upload_store(), bucket, key, upload_id, part_size, context) oss2.resumable_upload(bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024) result = bucket.get_object(key) self.assertEqual(content, result.read()) self.assertEqual( len(list(oss2.ObjectUploadIterator(self.bucket, key))), expected_unfinished) bucket.delete_object(key)
def __test_interrupt(self, content_size, failed_part_number, expected_unfinished=0, modify_record_func=None): bucket = random.choice([self.bucket, self.rsa_crypto_bucket, self.kms_crypto_bucket]) encryption_flag = isinstance(bucket, oss2.CryptoBucket) orig_upload_part = bucket.upload_part if encryption_flag: def upload_part(self, key, upload_id, part_number, data, progress_callback=None, headers=None, upload_context=None): if part_number == failed_part_number: raise RuntimeError else: return orig_upload_part(key, upload_id, part_number, data, progress_callback, headers, upload_context) else: def upload_part(self, key, upload_id, part_number, data, progress_callback=None, headers=None): if part_number == failed_part_number: raise RuntimeError else: return orig_upload_part(key, upload_id, part_number, data, progress_callback, headers) key = 'ResumableUpload-' + random_string(32) content = random_bytes(content_size) pathname = self._prepare_temp_file(content) if encryption_flag: with patch.object(oss2.CryptoBucket, 'upload_part', side_effect=upload_part, autospec=True) as mock_upload_part: self.assertRaises(RuntimeError, oss2.resumable_upload, bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024) else: with patch.object(oss2.Bucket, 'upload_part', side_effect=upload_part, autospec=True) as mock_upload_part: self.assertRaises(RuntimeError, oss2.resumable_upload, bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024) if modify_record_func: modify_record_func(oss2.resumable.make_upload_store(), bucket.bucket_name, key, pathname) oss2.resumable_upload(bucket, key, pathname, multipart_threshold=0, part_size=100 * 1024) self.assertEqual(len(list(oss2.ObjectUploadIterator(self.bucket, key))), expected_unfinished)