def test_determine_part_size(self): self.assertEqual(oss2.determine_part_size(oss2.defaults.part_size + 1), oss2.defaults.part_size) self.assertEqual(oss2.determine_part_size(1), 1) self.assertEqual(oss2.determine_part_size(1, oss2.defaults.part_size+1), 1) n = 10000 size = (oss2.defaults.part_size + 1) * n part_size = oss2.determine_part_size(size) self.assertTrue(n * part_size <= size) self.assertTrue(oss2.defaults.part_size < part_size)
def test_determine_part_size(self): self.assertEqual(oss2.determine_part_size(oss2.defaults.part_size + 1), oss2.defaults.part_size) self.assertEqual(oss2.determine_part_size(1), 1) self.assertEqual( oss2.determine_part_size(1, oss2.defaults.part_size + 1), 1) n = 10000 size = (oss2.defaults.part_size + 1) * n part_size = oss2.determine_part_size(size) self.assertTrue(n * part_size <= size) self.assertTrue(oss2.defaults.part_size < part_size)
def upload_chunk_file(self, cloud_file, file_to_upload): bucket = self.__connect_oss() total_size = os.path.getsize(file_to_upload) part_size = oss2.determine_part_size(total_size, preferred_size=self.__file_chunk_size) file_part_count = (total_size / part_size) + 1 upload_id = bucket.init_multipart_upload(cloud_file).upload_id chunk_information = [] logger.info("开始分片存储%s " % file_to_upload ) # TODO 并发上传 startTime = time.time() for chunkIdx in xrange(file_part_count): offset = part_size * chunkIdx bytes = min(part_size, total_size - offset) fp = FileChunkIO(file_to_upload, 'r', offset=offset, bytes=bytes) chunk_information.append(dict( bucket=bucket, cloud_file=cloud_file, upload_id=upload_id, part_number=chunkIdx+1, file_size_adapter=fp, )) muti_upload = [] for each in range(self.muti_upload_chunk_num): muti_upload.append(gevent.spawn(self.complicate_upload, chunk_information)) gevent.joinall(muti_upload) bucket.complete_multipart_upload(cloud_file, upload_id, self.__parts) endTime = time.time() spendTime = endTime - startTime logger.info("Upload file %s spent %f second." % (file_to_upload, spendTime))
def uploadFile(self): filelist = self.file_name() if filelist: try: for file in filelist: fileDir = file.split(self.dirname)[1] key = (self.dirname + fileDir).replace('\\', '/') total_size = os.path.getsize(file) # determine_part_size方法用来确定分片大小。 part_size = determine_part_size(total_size, preferred_size=100 * 1024) # 初始化分片。 upload_id = self.bucket.init_multipart_upload(key).upload_id parts = [] # 逐个上传分片。 with open(file, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) # SizedFileAdapter(fileobj, size)方法会生成一个新的文件对象,重新计算起始追加位置。 result = self.bucket.upload_part(key, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # 完成分片上传。 self.bucket.complete_multipart_upload(key, upload_id, parts) logging.info('upload file to yun ok') self.sendMail() except Exception as e: logging.info('upload file to yun error') logging.error(e)
def upload_chunk_file(self, cloud_file, file_to_upload): bucket = self.__connect_oss() total_size = os.path.getsize(file_to_upload) part_size = oss2.determine_part_size(total_size, preferred_size=self.__file_chunk_size) file_part_count = (total_size / part_size) + 1 upload_id = bucket.init_multipart_upload(cloud_file).upload_id parts = [] print "开始分片存储%s。。。。。" % file_to_upload startTime = time.time() with open(file_to_upload, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part(cloud_file, upload_id, part_number, oss2.SizedFileAdapter(fileobj, num_to_upload)) parts.append(oss2.models.PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 print "upload chunk %d" % (part_number - 1) bucket.complete_multipart_upload(cloud_file, upload_id, parts) endTime = time.time() spendTime = endTime - startTime print "Upload file spend %f second." % (spendTime)
def upload(self): psize = oss2.determine_part_size(self.__totalSize, preferred_size=self.__partSize) # 初始化分片 self.__uploadId = self.__bucket.init_multipart_upload(self.__key).upload_id startTime = time.time() expireSeconds = 2500 # 上传凭证有效期3000秒,提前刷新 # 逐个上传分片 with open(AliyunVodUtils.toUnicode(self.__fileName), 'rb') as fileObj: partNumber = 1 offset = 0 while offset < self.__totalSize: uploadSize = min(psize, self.__totalSize - offset) #logger.info("UploadPart, FilePath: %s, VideoId: %s, UploadId: %s, PartNumber: %s, PartSize: %s" % (self.__fileName, self.__videoId, self.__uploadId, partNumber, uploadSize)) result = self.__bucket.upload_part(self.__key, self.__uploadId, partNumber, SizedFileAdapter(fileObj,uploadSize)) #print(result.request_id) self.__finishedParts.append(PartInfo(partNumber, result.etag)) offset += uploadSize partNumber += 1 # 上传进度回调 self.__progressCallback(offset, self.__totalSize) # 检测上传凭证是否过期 nowTime = time.time() if nowTime - startTime >= expireSeconds: self.__bucket = self.__refreshAuthCallback(self.__videoId) startTime = nowTime # 完成分片上传 self.__bucket.complete_multipart_upload(self.__key, self.__uploadId, self.__finishedParts, headers=self.__headers) return result
def upload_multiple_object(self, bucket_name, obj_name, obj_file): try: bucket = oss2.Bucket(auth, url, bucket_name) total_size = os.path.getsize(obj_file) # Use determine_part_size to determine the part size. part_size = determine_part_size(total_size, preferred_size=100 * 1024) # Initialize a multipart upload event. upload_id = bucket.init_multipart_upload(obj_name).upload_id parts = [] # Upload parts one by one. with open(obj_file, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) # The SizedFileAdapter(fileobj, size) method generates a new object, and re-calculates the initial append location. result = bucket.upload_part( obj_name, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # Complete multipart upload. bucket.complete_multipart_upload(obj_name, upload_id, parts) # Verify the multipart upload. with open(obj_file, 'rb') as fileobj: assert bucket.get_object(obj_name).read() == fileobj.read() status = "success" except Exception as e: status = str(e) return status
def upload(self, upload_path, filepath): """ upload_path 文件上传后的完整路径包括本身 filepath 本地文件路径 """ key = upload_path filename = filepath total_size = os.path.getsize(filename) # determine_part_size方法用来确定分片大小。 part_size = determine_part_size(total_size, preferred_size=100 * 1024) # 初始化分片。 upload_id = self.bucket.init_multipart_upload(key).upload_id parts = [] # 逐个上传分片。 with open(filename, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) # SizedFileAdapter(fileobj, size)方法会生成一个新的文件对象,重新计算起始追加位置。 result = self.bucket.upload_part( key, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # 完成分片上传。 self.bucket.complete_multipart_upload(key, upload_id, parts)
def main(key, filename): auth = oss2.Auth('accessId', 'accessSecret') bucket = oss2.Bucket(auth, 'endpoint.aliyuncs.com', 'bucket') total_size = os.path.getsize(filename) part_size = oss2.determine_part_size(total_size, preferred_size=10 * 1024 * 1024) # init upload_id = bucket.init_multipart_upload(key).upload_id parts = [] print 'start to upload {} with id {}'.format(filename, upload_id) # upload with open(filename, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part( key, upload_id, part_number, oss2.SizedFileAdapter(fileobj, num_to_upload)) parts.append(oss2.models.PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # complete bucket.complete_multipart_upload(key, upload_id, parts) print 'done!'
def large_file_to_oss(id, file, type_name='courses'): try: file_name = '' if file: _, file_ext = os.path.splitext(file.name) md = hashlib.md5() md.update((str(time.time()) + file.name).encode('utf8')) file_name = type_name + '/' + str(id) + '/' + md.hexdigest() + file_ext oss_file_name = os.path.join(settings.ALIYUN_OSS_DIRECTORY_PREFIX, file_name) bucket = aliyun_oss() key = oss_file_name filename = file total_size = filename.size part_size = determine_part_size(total_size, preferred_size=100 * 1024) # 初始化分片 upload_id = bucket.init_multipart_upload(key).upload_id parts = [] # 逐个上传分片 part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part(key, upload_id, part_number, SizedFileAdapter(file, num_to_upload)) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # 完成分片上传 bucket.complete_multipart_upload(key, upload_id, parts) # # 验证一下 # # assert bucket.get_object(key).read() == file.read() return file_name except Exception: logger.error("Exceptions: {}".format(sys.exc_info()[0]))
def persist_file(self, path, buf, info, meta=None, headers=None): # 首先可以用帮助函数设定分片大小,设我们期望的分片大小为128KB total_size = len(buf) part_size = oss2.determine_part_size(total_size, preferred_size=128 * 1024) # 初始化分片上传,得到Upload ID。接下来的接口都要用到这个Upload ID。 key = os.path.join(self.objectPath, info) upload_id = self.bucket.init_multipart_upload(key).upload_id # 逐个上传分片 # 其中oss2.SizedFileAdapter()把fileobj转换为一个新的文件对象,新的文件对象可读的长度等于size_to_upload parts = [] part_number = 1 offset = 0 while offset < total_size: size_to_upload = min(part_size, total_size - offset) result = self.bucket.upload_part(key, upload_id, part_number, oss2.SizedFileAdapter(buf, size_to_upload)) parts.append(oss2.models.PartInfo(part_number, result.etag, size=size_to_upload, part_crc=result.crc)) offset += size_to_upload part_number += 1 # 完成分片上传 self.bucket.complete_multipart_upload(key, upload_id, parts)
def OssSliceUpload(): accessKeyId = sys.argv[1] accessSecret = sys.argv[2] EndPoint = sys.argv[3] Bucket = sys.argv[4] RemoteFile = sys.argv[5] LocalFile = sys.argv[6] auth = oss2.Auth(accessKeyId, accessSecret) bucket = oss2.Bucket(auth, EndPoint, Bucket) key = RemoteFile filename = LocalFile total_size = os.path.getsize(filename) part_size = determine_part_size(total_size, preferred_size=100 * 1024) # 初始化分片 upload_id = bucket.init_multipart_upload(key).upload_id parts = [] # 逐个上传分片 with open(filename, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part( key, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # 完成分片上传 bucket.complete_multipart_upload(key, upload_id, parts) # 验证一下 with open(filename, 'rb') as fileobj: assert bucket.get_object(key).read() == fileobj.read()
def upload(self): psize = oss2.determine_part_size(self.__totalSize, preferred_size=self.__partSize) # 初始化分片 self.__uploadId = self.__bucket.init_multipart_upload( self.__key).upload_id # 逐个上传分片 with open(AliyunVodUtils.toUnicode(self.__fileName), 'rb') as fileObj: partNumber = 1 offset = 0 while offset < self.__totalSize: uploadSize = min(psize, self.__totalSize - offset) logger.info( "UploadPart, FilePath: %s, VideoId: %s, UploadId: %s, PartNumber: %s, PartSize: %s" % (self.__fileName, self.__videoId, self.__uploadId, partNumber, uploadSize)) result = self.__bucket.upload_part( self.__key, self.__uploadId, partNumber, SizedFileAdapter(fileObj, uploadSize)) self.__finishedParts.append(PartInfo(partNumber, result.etag)) offset += uploadSize partNumber += 1 # 完成分片上传 self.__bucket.complete_multipart_upload(self.__key, self.__uploadId, self.__finishedParts, headers=self.__headers) return result
def test_upload_part(self): key = 'requestpayment-test-upload-part-object' filename = key + '.txt' content = b'1' * 1024 * 1024 headers = dict() headers[OSS_REQUEST_PAYER] = "requester" content = random_bytes(1024 * 1024) with open(filename, 'wb') as f: f.write(content) total_size = os.path.getsize(filename) # Set part size part_size = determine_part_size(total_size, preferred_size=(100*1024)) # Init multipart without payer setting, should be failed. self.assertRaises(oss2.exceptions.AccessDenied, self.payer_bucket.init_multipart_upload, key) # Init multipart with payer setting, should be successful. upload_id = self.payer_bucket.init_multipart_upload(key, headers=headers).upload_id parts = [] # Upload part without payer setting, should be failed. with open(filename, 'rb') as fileobj: part_number = 1 num_to_upload = part_size self.assertRaises(oss2.exceptions.AccessDenied, self.payer_bucket.upload_part, key, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)) # Upload part with payer setting, should be successful. with open(filename, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = self.payer_bucket.upload_part(key, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload), headers=headers) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # Complete multipart upload without payer setting, should be failed. self.assertRaises(oss2.exceptions.AccessDenied, self.payer_bucket.complete_multipart_upload, key, upload_id, parts) # Complete multipart upload with payer setting, should be successful. result = self.payer_bucket.complete_multipart_upload(key, upload_id, parts, headers=headers) self.assertEqual(result.status, 200) self.bucket.delete_object(key) os.remove(filename)
def oss_upload(osskey, filename, uploadpath, de_suffix=True): try: # 当 de_suffix 是Ture,使用正则把文件的后缀去掉 if de_suffix is True: osskey = re.match(r'\w+', filename).group() # 阿里云主账号AccessKey拥有所有API的访问权限,风险很高。强烈建议您创建并使用RAM账号进行API访问或日常运维,请登录 https://ram.console.aliyun.com 创建RAM账号。 auth = oss2.Auth(ACCESSKEYID, ACCESSKEYSECRET) # Endpoint以杭州为例,其它Region请按实际情况填写。 bucket = oss2.Bucket(auth, ENDPOINT, BUCKETNAME) key = uploadpath + osskey filename = filename total_size = os.path.getsize(filename) # determine_part_size方法用来确定分片大小。 part_size = determine_part_size(total_size, preferred_size=10240 * 1000) # 初始化分片。 upload_id = bucket.init_multipart_upload(key).upload_id parts = [] # 逐个上传分片。 with open(filename, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) # SizedFileAdapter(fileobj, size)方法会生成一个新的文件对象,重新计算起始追加位置。 result = bucket.upload_part( key, upload_id, part_number, SizedFileAdapter( fileobj, num_to_upload, ), ) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # 完成分片上传 bucket.complete_multipart_upload(key, upload_id, parts) except Exception as f: Print.error(f)
def test_upload_part(self): key = 'traffic-limit-test-resumble-upload-object' # Create tmp file 2MB file_name = self._prepare_temp_file_with_size(OBJECT_SIZE_1MB * 2) total_size = os.path.getsize(file_name) # Determine part size is 1MB part_size = determine_part_size(total_size, preferred_size=(1024 * 1024)) # Init part upload_id = self.bucket.init_multipart_upload(key).upload_id parts = [] headers = dict() headers[OSS_TRAFFIC_LIMIT] = str(LIMIT_100KB) # Upload part with open(file_name, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) # Upload part with traffic limit setting start_time_sec = int(time.time()) result = self.bucket.upload_part(key, upload_id, part_number, SizedFileAdapter( fileobj, num_to_upload), headers=headers) end_time_sec = int(time.time()) # Calculate expensed time expense_time_sec = end_time_sec - start_time_sec # Theoretical time is 1MB/100KB = 10s, set the minimum theoretical time to 10*0.7s theoretical_exepnse_min = 10 * 0.7 # Compare to minimum theoretical time self.assertEqual((expense_time_sec > theoretical_exepnse_min), True) self.assertEqual((expense_time_sec > theoretical_exepnse_min), True) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 result = self.bucket.complete_multipart_upload(key, upload_id, parts) self.assertEqual(result.status, 200)
def test_upload_part_copy(self): src_object_name = 'requestpayment-test-upload-part-copy-src' dest_object_name = 'requestpayment-test-upload-part-copy-dest' content = b'a' * 1024 * 1024 headers = dict() headers[OSS_REQUEST_PAYER] = "requester" self.bucket.put_object(src_object_name, content) # Get src object size head_info = self.bucket.head_object(src_object_name) total_size = head_info.content_length self.assertEqual(total_size, 1024 * 1024) # Set part size part_size = determine_part_size(total_size, preferred_size=(100*1024)) upload_id = self.payer_bucket.init_multipart_upload(dest_object_name, headers=headers).upload_id parts = [] # Upload part copy without payer setting, should be failed. part_number = 1 offset = 0 num_to_upload = min(part_size, total_size - offset) end = offset + num_to_upload - 1; self.assertRaises(oss2.exceptions.AccessDenied, self.payer_bucket.upload_part_copy, self.payer_bucket.bucket_name, src_object_name, (offset, end), dest_object_name, upload_id, part_number) # Upload part copy with payer setting, should be successful. part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) end = offset + num_to_upload - 1; result = self.payer_bucket.upload_part_copy(self.payer_bucket.bucket_name, src_object_name, (offset, end), dest_object_name, upload_id, part_number, headers=headers) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # Complete multipart upload with payer setting, should be successful. result = self.payer_bucket.complete_multipart_upload(dest_object_name, upload_id, parts, headers=headers) self.bucket.delete_object(src_object_name) self.bucket.delete_object(dest_object_name)
def test_upload_success(test_bucket_access, test_create_file): bucket = test_bucket_access bucket_stat = bucket.get_bucket_stat() count_before_upload = bucket_stat.object_count current_app.logger.info('storage: ' + str(bucket_stat.storage_size_in_bytes)) current_app.logger.info('object count: ' + str(bucket_stat.object_count)) current_app.logger.info('multi part upload count: ' + str(bucket_stat.multi_part_upload_count)) filename = test_create_file # 也可以直接调用分片上传接口。 # 首先可以用帮助函数设定分片大小,设我们期望的分片大小为128KB import os import oss2 total_size = os.path.getsize(filename) current_app.logger.info('filesize: ' + str(total_size)) part_size = oss2.determine_part_size(total_size, preferred_size=128 * 1024) # 初始化分片上传,得到Upload ID。接下来的接口都要用到这个Upload ID。 key = '{0}.txt'.format(random_string(10)) upload_id = bucket.init_multipart_upload(key).upload_id # 逐个上传分片 # 其中oss2.SizedFileAdapter()把fileobj转换为一个新的文件对象,新的文件对象可读的长度等于size_to_upload with open(filename, 'rb') as fileobj: parts = [] part_number = 1 offset = 0 while offset < total_size: size_to_upload = min(part_size, total_size - offset) result = bucket.upload_part( key, upload_id, part_number, oss2.SizedFileAdapter(fileobj, size_to_upload)) parts.append( oss2.models.PartInfo(part_number, result.etag, size=size_to_upload, part_crc=result.crc)) offset += size_to_upload part_number += 1 # 完成分片上传 bucket.complete_multipart_upload(key, upload_id, parts) count_after_upload = bucket.get_bucket_stat().object_count assert count_before_upload < count_after_upload
def upload(oss_task_zip, zip_task_file, endpoint, bucket_name): dlog.debug(f"debug: upload: oss_task_zip:{oss_task_zip}; zip_task_file:{zip_task_file}") bucket = _get_oss_bucket(endpoint, bucket_name) total_size = os.path.getsize(zip_task_file) part_size = determine_part_size(total_size, preferred_size=1000 * 1024) upload_id = bucket.init_multipart_upload(oss_task_zip).upload_id parts = [] with open(zip_task_file, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part(oss_task_zip, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # result = bucket.complete_multipart_upload(oss_task_zip, upload_id, parts) result = bucket.complete_multipart_upload(oss_task_zip, upload_id, parts) # print('debug:upload_result:', result, dir()) return result
def push_by_piece(self, path, name): path = Path(path) total_size = path.stat().st_size part_size = determine_part_size(total_size, preferred_size=1024 * 1024) upload_id = self.bucket.init_multipart_upload(name).upload_id parts = [] with open(path, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) # SizedFileAdapter(fileobj, size)方法会生成一个新的文件对象,重新计算起始追加位置。 result = self.bucket.upload_part(name, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 headers = {'Content-MD5': self.get_md5(path)} self.bucket.complete_multipart_upload(name, upload_id, parts, headers=headers)
def uploadFile2OssByPart(self, file, objPath): # 分片上传方法 ''' 把文件对象分片上传oss :param file: 文件对象 :param objPath: 上传到oss对应的文件名(路径) :return: ''' totalSize = file.size # determine_part_size方法用于确定分片大小。 partSize = oss2.determine_part_size(totalSize, preferred_size=PREFERRED_SIZE) uploadId = self.bucket.init_multipart_upload(objPath).upload_id parts = [] # 分片上传 executor = ThreadPoolExecutor(max_workers=1) allTask = [] partNumber = 1 offset = 0 while offset < totalSize: numToUpload = min(partSize, totalSize - offset) print(partNumber, numToUpload) ## 一般上传 # 调用SizedFileAdapter(file, size)方法会生成一个新的文件对象,重新计算起始追加位置。 result = self.bucket.upload_part( objPath, uploadId, partNumber, oss2.SizedFileAdapter(file, numToUpload)) parts.append(oss2.models.PartInfo(partNumber, result.etag)) ## 多线程上传 # allTask.append(executor.submit(_uploadPart, partNumber, numToUpload)) # offset += numToUpload # partNumber += 1 # 完成分片上传。 wait(allTask, return_when=ALL_COMPLETED) resultList = [future.result() for future in as_completed(allTask)] for data in sorted(resultList, key=lambda x: x[0]): # 重排序按正常数据加入parts partNumber, result = data[0], data[1] print(partNumber) parts.append(oss2.models.PartInfo(partNumber, result.etag)) self.bucket.complete_multipart_upload(objPath, uploadId, parts) return getOssSavePath(objPath)
def main(args): filename = args.upload_file key = os.path.split(filename)[1] for param in (access_key_id, access_key_secret, bucket_name, endpoint): assert '<' not in param, '请设置参数:' + param # 创建Bucket对象,所有Object相关的接口都可以通过Bucket对象来进行 bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name) total_size = os.path.getsize(filename) part_size = determine_part_size(total_size, preferred_size=100000 * 1024) # 初始化分片 print('OSS initializing:') encode_md5 = calculate_file_md5(filename) print('%s s md5 is %s' % (filename, encode_md5)) upload_id = bucket.init_multipart_upload(key).upload_id parts = [] # 逐个上传分片 print('Seperate the file batch and uploading:') with open(filename, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part(key, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload), progress_callback=percentage) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # 完成分片上传 print('Complete the uploading task:') result = bucket.complete_multipart_upload(key, upload_id, parts) print('upload results: %i' % result) print ('upload complete with the file %s' % key)
def upload(filename, key): total_size = os.path.getsize(filename) part_size = determine_part_size(total_size, preferred_size=100 * 1024) END_POINT = 'oss-cn-shanghai.aliyuncs.com' AUTH = Auth('LTAIStvC4wpBWRVG', 'BNXtvOz82JjzlSLjPBdQJyEUpXi4PD') BUCKET_NAME = "yunbeifeng" bucket = Bucket(AUTH, END_POINT, bucket_name=BUCKET_NAME) upload_id = bucket.init_multipart_upload(key).upload_id parts = [] with open(filename, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part( key, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 bucket.complete_multipart_upload(key, upload_id, parts)
def copy_data(self, from_key, to_key): """复制文件""" total_size = self.bucket.head_object(from_key).content_length part_size = determine_part_size(total_size, preferred_size=100 * 1024) # 初始化分片 upload_id = self.bucket.init_multipart_upload(to_key).upload_id parts = [] # 逐个分片拷贝 part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) byte_range = (offset, offset + num_to_upload - 1) result = self.bucket.upload_part_copy(self.bucket.bucket_name, from_key , byte_range, to_key, upload_id, part_number) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 # 完成分片上传 self.bucket.complete_multipart_upload(to_key, upload_id, parts)
def _store_in_thread(self, file): # 首先可以用帮助函数设定分片大小,设我们期望的分片大小为128KB total_size = os.path.getsize(file) part_size = oss2.determine_part_size(total_size, preferred_size=128 * 1024) # 初始化分片上传,得到Upload ID。接下来的接口都要用到这个Upload ID。 key = file.replace('../', '') upload_id = self.bucket.init_multipart_upload(key).upload_id # 逐个上传分片 # 其中oss2.SizedFileAdapter()把fileobj转换为一个新的文件对象,新的文件对象可读的长度等于size_to_upload with open(file, 'rb') as fileobj: parts = [] part_number = 1 offset = 0 while offset < total_size: size_to_upload = min(part_size, total_size - offset) result = self.bucket.upload_part( key, upload_id, part_number, oss2.SizedFileAdapter(fileobj, size_to_upload)) parts.append( oss2.models.PartInfo(part_number, result.etag, size=size_to_upload, part_crc=result.crc)) offset += size_to_upload part_number += 1 # 完成分片上传 self.bucket.complete_multipart_upload(key, upload_id, parts) # 验证一下 with open(file, 'rb') as fileobj: assert self.bucket.get_object(key).read() == fileobj.read()
def test_multipart_upload(self): auth = oss2.Auth(OSS_ID, OSS_SECRET) bucket_name = OSS_BUCKET + "-test-multipart-upload-data-encryption" bucket = oss2.Bucket(auth, self.endpoint, bucket_name) bucket.create_bucket() key = 'data-encryption-test-upload-part-object' filename = self._prepare_temp_file_with_size(1024 * 1024) headers = dict() headers[OSS_SERVER_SIDE_ENCRYPTION] = SERVER_SIDE_ENCRYPTION_KMS headers[OSS_SERVER_SIDE_DATA_ENCRYPTION] = KMS_DATA_ENCRYPTION_SM4 total_size = os.path.getsize(filename) # Set part size part_size = determine_part_size(total_size, preferred_size=(100*1024)) # Init multipart with encryption headers. result = bucket.init_multipart_upload(key, headers=headers) ret_sse_algo = result.headers.get(OSS_SERVER_SIDE_ENCRYPTION) ret_data_algo = result.headers.get(OSS_SERVER_SIDE_DATA_ENCRYPTION) ret_kms_key_id = result.headers.get(OSS_SERVER_SIDE_ENCRYPTION_KEY_ID) self.assertEqual(SERVER_SIDE_ENCRYPTION_KMS, ret_sse_algo) self.assertEqual(KMS_DATA_ENCRYPTION_SM4, ret_data_algo) self.assertIsNotNone(ret_kms_key_id) kms_key_id = ret_kms_key_id upload_id = result.upload_id parts = [] # Upload part with the encyrption headers will be failed. headers = dict() headers[OSS_SERVER_SIDE_ENCRYPTION] = SERVER_SIDE_ENCRYPTION_KMS headers[OSS_SERVER_SIDE_DATA_ENCRYPTION] = KMS_DATA_ENCRYPTION_SM4 with open(filename, 'rb') as fileobj: part_number = 1 num_to_upload = part_size self.assertRaises(oss2.exceptions.InvalidArgument, bucket.upload_part, key, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload), headers=headers) # Upload part with none encryption headers. with open(filename, 'rb') as fileobj: part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part(key, upload_id, part_number, SizedFileAdapter(fileobj, num_to_upload)) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 ret_sse_algo = result.headers.get(OSS_SERVER_SIDE_ENCRYPTION) ret_data_algo = result.headers.get(OSS_SERVER_SIDE_DATA_ENCRYPTION) ret_kms_key_id = result.headers.get(OSS_SERVER_SIDE_ENCRYPTION_KEY_ID) self.assertEqual(SERVER_SIDE_ENCRYPTION_KMS, ret_sse_algo) self.assertEqual(KMS_DATA_ENCRYPTION_SM4, ret_data_algo) self.assertEqual(kms_key_id, ret_kms_key_id) # Complete multipart upload with encryption headers. self.assertRaises(oss2.exceptions.InvalidArgument, bucket.complete_multipart_upload, key, upload_id, parts, headers=headers) # Complete multipart upload with none encryption headers. result = bucket.complete_multipart_upload(key, upload_id, parts) self.assertEqual(result.status, 200) ret_sse_algo = result.headers.get(OSS_SERVER_SIDE_ENCRYPTION) ret_data_algo = result.headers.get(OSS_SERVER_SIDE_DATA_ENCRYPTION) ret_kms_key_id = result.headers.get(OSS_SERVER_SIDE_ENCRYPTION_KEY_ID) self.assertEqual(SERVER_SIDE_ENCRYPTION_KMS, ret_sse_algo) self.assertEqual(KMS_DATA_ENCRYPTION_SM4, ret_data_algo) self.assertEqual(kms_key_id, ret_kms_key_id) bucket.delete_object(key)
def copy(self, src, dst): raw_src, raw_dst = str(src), str(dst) try: cloud_src = src.startswith('oss://') cloud_dst = dst.startswith('oss://') if not cloud_src and not cloud_dst: return super().copy(src, dst) if src == dst: return # download if cloud_src and not cloud_dst: target_dir, _ = os.path.split(dst) if not os.path.exists(target_dir): os.makedirs(target_dir) bucket, src = self._split(src) obj = bucket.get_object(src) if obj.content_length > 100 * 1024 ** 2: # 100M with oss_progress('downloading') as callback: bucket.get_object_to_file(src, dst, progress_callback=callback) else: bucket.get_object_to_file(src, dst) return bucket, dst = self._split(dst) # upload if cloud_dst and not cloud_src: src_size = os.stat(src).st_size if src_size > 5 * 1024 ** 3: # 5G raise RuntimeError(f'A file > 5G cannot be uploaded to OSS. Please split your file first.\n{src}') if src_size > 100 * 1024 ** 2: # 100M with oss_progress('uploading') as callback: bucket.put_object_from_file(dst, src, progress_callback=callback) else: bucket.put_object_from_file(dst, src) return # copy between oss paths src_bucket, src = self._split(src) total_size = src_bucket.head_object(src).content_length if src_bucket.get_bucket_location().location != bucket.get_bucket_location().location: import tempfile local_tmp = os.path.join(tempfile.gettempdir(), src) self.copy(f'oss://{src_bucket.bucket_name}/{src}', local_tmp) self.copy(local_tmp, f'oss://{bucket.bucket_name}/{dst}') self.remove(local_tmp) return if total_size < 1024 ** 3 or src_bucket != bucket: # 1GB bucket.copy_object(src_bucket.bucket_name, src, dst) else: # multipart copy from oss2.models import PartInfo from oss2 import determine_part_size part_size = determine_part_size(total_size, preferred_size=100 * 1024) upload_id = bucket.init_multipart_upload(dst).upload_id parts = [] part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) byte_range = (offset, offset + num_to_upload - 1) result = bucket.upload_part_copy(bucket.bucket_name, src, byte_range, dst, upload_id, part_number) parts.append(PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 bucket.complete_multipart_upload(dst, upload_id, parts) except Exception as e: print("haha") print("{}".format(e)) print("Copy failed because oss auth not fully opened. Using first download then upload...") try: self.download(raw_src, ".easy_distill_tmp_file") self.upload(".easy_distill_tmp_file", raw_dst) print("Copying done") except Exception as e: print("{}".format(e))
kms_key_id = result.headers.get(OSS_SERVER_SIDE_ENCRYPTION_KEY_ID) print('sse_algorithm:', sse_algo) print('data_algorithm:', data_algo) print('kms_key_id:', kms_key_id) # 读取上传内容 object_stream = bucket.get_object(key) print(object_stream.read()) # ##########以下是使用分片上传接口上传文件时单独指定文件的服务端加密方式的示例############ key = 'test-upload_file' filename = '<yourLocalFile>' total_size = os.path.getsize(filename) # determine_part_size方法用来确定分片大小。 part_size = determine_part_size(total_size, preferred_size=100 * 1024) # 在headers中指定加密方式 headers = dict() # 使用OSS server端SM4加密算法 headers[OSS_SERVER_SIDE_ENCRYPTION] = SERVER_SIDE_ENCRYPTION_SM4 # 初始化分片时指定文件在服务端端加密类型 result = bucket.init_multipart_upload(key, headers=headers) sse_algo = result.headers.get(OSS_SERVER_SIDE_ENCRYPTION) data_algo = result.headers.get(OSS_SERVER_SIDE_DATA_ENCRYPTION) kms_key_id = result.headers.get(OSS_SERVER_SIDE_ENCRYPTION_KEY_ID) print('sse_algorithm:', sse_algo) print('data_algorithm:', data_algo) print('kms_key_id:', kms_key_id)
with open(filename, 'wb') as fileobj: fileobj.write(content) # 断点续传一:因为文件比较小(小于oss2.defaults.multipart_threshold), # 所以实际上用的是oss2.Bucket.put_object oss2.resumable_upload(bucket, 'remote-normal.txt', filename) # 断点续传二:为了展示的需要,我们指定multipart_threshold可选参数,确保使用分片上传 oss2.resumable_upload(bucket, 'remote-multipart.txt', filename, multipart_threshold=100 * 1024) # 也可以直接调用分片上传接口。 # 首先可以用帮助函数设定分片大小,设我们期望的分片大小为128KB total_size = os.path.getsize(filename) part_size = oss2.determine_part_size(total_size, preferred_size=128 * 1024) # 初始化分片上传,得到Upload ID。接下来的接口都要用到这个Upload ID。 key = 'remote-multipart2.txt' upload_id = bucket.init_multipart_upload(key).upload_id # 逐个上传分片 # 其中oss2.SizedFileAdapter()把fileobj转换为一个新的文件对象,新的文件对象可读的长度等于num_to_upload with open(filename, 'rb') as fileobj: parts = [] part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part(key, upload_id, part_number, oss2.SizedFileAdapter(fileobj, num_to_upload))
fileobj.write(content) # 断点续传一:因为文件比较小(小于oss2.defaults.multipart_threshold), # 所以实际上用的是oss2.Bucket.put_object oss2.resumable_upload(bucket, 'remote-normal.txt', filename) # 断点续传二:为了展示的需要,我们指定multipart_threshold可选参数,确保使用分片上传 oss2.resumable_upload(bucket, 'remote-multipart.txt', filename, multipart_threshold=100 * 1024) # 也可以直接调用分片上传接口。 # 首先可以用帮助函数设定分片大小,设我们期望的分片大小为128KB total_size = os.path.getsize(filename) part_size = oss2.determine_part_size(total_size, preferred_size=128 * 1024) # 初始化分片上传,得到Upload ID。接下来的接口都要用到这个Upload ID。 key = 'remote-multipart2.txt' upload_id = bucket.init_multipart_upload(key).upload_id # 逐个上传分片 # 其中oss2.SizedFileAdapter()把fileobj转换为一个新的文件对象,新的文件对象可读的长度等于num_to_upload with open(filename, 'rb') as fileobj: parts = [] part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part( key, upload_id, part_number,
def uploadFile(self): """ Function check if cache file with uploading information is exist and try to continue uplod otherwise start new file uploading to OSS :return: """ total_size = os.stat(self.queue_object.path).st_size part_size = oss2.determine_part_size(total_size, preferred_size=2 * 1024 * 1024) try: with open(self.pkl_path, 'rb') as input_file: # We try to open file with title of uploading key storied_object = pickle.load(input_file) # if file is open means upload was not finished parts = storied_object.parts upload_id = storied_object.upload_id bucket_name = storied_object.bucket_name endpoint = storied_object.endpoint bucket = oss2.Bucket(self._auth, endpoint, bucket_name) # offset = storied_object.offset except IOError: bucket = oss2.Bucket(self._auth, self.queue_object.endpoint, self.queue_object.bucket_name) upload_id = bucket.init_multipart_upload(self.queue_object.key).upload_id storied_object = UploadingObject() uploade_parts = bucket.list_multipart_uploads() for part in uploade_parts.upload_list: if part.upload_id == upload_id: with open(self.pkl_path, 'wb') as output_file: storied_object.key = self.queue_object.key storied_object.bucket_name = self.queue_object.bucket_name storied_object.endpoint = self.queue_object.endpoint storied_object.path = self.queue_object.path storied_object.upload_id = upload_id storied_object.initiate_date = part.initiation_date pickle.dump(storied_object, output_file, pickle.HIGHEST_PROTOCOL) parts = [] with open(self.queue_object.path, 'rb') as fileobj: while storied_object.offset < total_size and not self.stoprequest.isSet(): # print storied_object.part_number # print storied_object.parts num_to_upload = min(part_size, total_size - storied_object.offset) upload_content = oss2.SizedFileAdapter(fileobj, num_to_upload) try: result = bucket.upload_part(self.queue_object.key, upload_id, storied_object.part_number, upload_content) except NoSuchUpload: print "\n ==== Not finished upload not exist on OSS bucket ====" print " Clean local cache, and update uploading queue" os.remove(self.pkl_path) raise UploadIdNotInBucket("Upload id is not in bucket") # Append directly to class is didn't work with "pickle" parts.append(oss2.models.PartInfo(storied_object.part_number, result.etag)) storied_object.parts = parts if num_to_upload == part_size: percentage = str(self._percentage(num_to_upload * storied_object.part_number, total_size)) else: percentage = 'Complete' # print percentage # logging.debug("File: %s => Bucket: %s - %s", key, bucket_name, percentage) # print "File: {0} => Bucket: {1} - {2}".format(self.queue_object.key.encode("utf-8"), # self.queue_object.bucket_name, # percentage) sys.stdout.write("\rFile: {0} => Bucket: {1} - {2}".format(self.queue_object.key.encode("utf-8"), self.queue_object.bucket_name, percentage)).splitlines() sys.stdout.flush() storied_object.offset += num_to_upload storied_object.part_number += 1 with open(self.pkl_path, 'wb') as output_file: pickle.dump(storied_object, output_file, pickle.HIGHEST_PROTOCOL) if not self.stoprequest.isSet(): bucket.complete_multipart_upload(self.queue_object.key, upload_id, parts) os.remove(self.pkl_path)
def handler(environ, start_response): context = environ['fc.context'] request_uri = environ['fc.request_uri'] for k, v in environ.items(): if k.startswith("HTTP_"): # process custom request headers print(k, v) pass # get request body try: request_body_size = int(environ.get('CONTENT_LENGTH', 0)) except (ValueError): request_body_size = 0 request_body = environ['wsgi.input'].read(request_body_size) request_body = urllib.unquote(request_body).decode('utf8') # get request method request_method = environ['REQUEST_METHOD'] if request_method != 'POST': status = '400 Bad Request' response_headers = [('Content-type', 'application/json')] start_response(status, response_headers) data = json.dumps({"error": "invalid request method."}) return [data] # print request body print('request_body: {}'.format(request_body)) request_body_json = json.loads(request_body) creds = context.credentials auth = oss2.StsAuth(creds.accessKeyId, creds.accessKeySecret, creds.securityToken) items = request_body_json.get("items") print("[DEBUG] items: {0}".format(items)) # zip name re_code = request_body_json.get("re_code") is_full = request_body_json.get("full") uuid = request_body_json.get("uuid") tmpdir = '/tmp/download/' os.system("rm -rf /tmp/*") os.mkdir(tmpdir) # download for item in items: print("[DEBUG] item: {}".format(item)) oss_protocol = item.get("protocol") oss_bucket_name = item.get("bucket") oss_endpoint = item.get("endpoint") file_path = item.get("path") file_original_name = item.get("original_name") bucket = oss2.Bucket(auth, oss_endpoint, oss_bucket_name) bucket.get_object_to_file(file_path, tmpdir + file_original_name) #zip file zipname = '/tmp/' + re_code + '.zip' make_zip(tmpdir, zipname) #upload total_size = os.path.getsize(zipname) part_size = oss2.determine_part_size(total_size, preferred_size=128 * 1024) if is_full: zip_path = 'full-archive/' + re_code + '/' + uuid + '.zip' else: zip_path = 'custom-archive/' + re_code + '/' + uuid + '.zip' # use the last bucket to upload zip package upload_id = bucket.init_multipart_upload(zip_path).upload_id with open(zipname, 'rb') as fileobj: parts = [] part_number = 1 offset = 0 while offset < total_size: num_to_upload = min(part_size, total_size - offset) result = bucket.upload_part( zip_path, upload_id, part_number, oss2.SizedFileAdapter(fileobj, num_to_upload)) parts.append(oss2.models.PartInfo(part_number, result.etag)) offset += num_to_upload part_number += 1 bucket.complete_multipart_upload(zip_path, upload_id, parts) zip_meta = bucket.head_object(zip_path) zip_content_type = zip_meta.headers.get('Content-Type') status = '200 OK' response_headers = [('Content-type', 'application/json')] start_response(status, response_headers) url = oss_protocol + "://" + oss_bucket_name + "." + oss_endpoint + "/" + zip_path data = json.dumps({ "host": url, "protocol": oss_protocol, "bucket": oss_bucket_name, "endpoint": oss_endpoint, "path": zip_path, "original_name": re_code + ".zip", "type": zip_content_type, }) return [data]