# SuperFile step 1: init multi-upload upload_id = bos_client.initiate_multipart_upload(bucket_name, key).upload_id upload_id_about = bos_client.initiate_multipart_upload(bucket_name, key + "_about").upload_id # SuperFile step 2: upload file part by part left_size = os.path.getsize(file_name) offset = 0 part_number = 1 part_list = [] while left_size > 0: part_size = 5 * 1024 * 1024 if left_size < part_size: part_size = left_size response = bos_client.upload_part_from_file( bucket_name, key, upload_id, part_number, part_size, file_name, offset) left_size -= part_size offset += part_size # your should store every part number and etag to invoke complete multi-upload part_list.append({ "partNumber": part_number, "eTag": response.metadata.etag }) part_number += 1 # list multi-uploads response = bos_client.list_multipart_uploads(bucket_name) for upload in response.uploads: __logger.debug("[Sample] list multi-uploads, upload_id:%s", upload.upload_id) # list parts
key).upload_id upload_id_about = bos_client.initiate_multipart_upload( bucket_name, key + "_about").upload_id # SuperFile step 2: upload file part by part left_size = os.path.getsize(file_name) offset = 0 part_number = 1 part_list = [] while left_size > 0: part_size = 5 * 1024 * 1024 if left_size < part_size: part_size = left_size response = bos_client.upload_part_from_file(bucket_name, key, upload_id, part_number, part_size, file_name, offset) left_size -= part_size offset += part_size # your should store every part number and etag to invoke complete multi-upload part_list.append({ "partNumber": part_number, "eTag": response.metadata.etag }) part_number += 1 # list multi-uploads response = bos_client.list_multipart_uploads(bucket_name) for upload in response.uploads: __logger.debug("[Sample] list multi-uploads, upload_id:%s", upload.upload_id)
def main(): """ main函数 :return: """ if len(sys.argv) != 3: print "参数个数不正确,请执行 sh test.sh 文件路径 Token" exit(0) path = sys.argv[1] token = sys.argv[2] if not os.path.exists(path): print "文件:" + path + " 不存在,请确认文件路径参数是否正确" exit(0) code, acl = bos_acl() if code != 200: print "服务异常,请稍候再试" exit(0) acl = json.loads(acl) if acl['errorCode'] != 0: print "服务异常,请稍候再试" exit(0) ak = unicode2str(acl['result']['accessKeyId']) sk = unicode2str(acl['result']['secretAccessKey']) bos_token = unicode2str(acl['result']['sessionToken']) config = BceClientConfiguration( credentials=BceCredentials(ak, sk), endpoint=unicode2str("http://bj.bcebos.com"), security_token=bos_token) bos_client = BosClient(config) bucket_name = unicode2str(acl['result']['bucketName']) object_key = unicode2str(acl['result']['fileKey']) token_status_code, token_data = check_token(token) if token_status_code != 200: print "服务异常,请稍候再试" exit(0) token_data = json.loads(token_data) if token_data['errorCode'] != 0: print token_data['errorMsg'] exit(0) print "开始提交" upload_id = bos_client.initiate_multipart_upload(bucket_name, object_key).upload_id left_size = os.path.getsize(path) # left_size用于设置分块开始位置 # 设置分块的开始偏移位置 offset = 0 part_number = 1 part_list = [] total_size = left_size while left_size > 0: # 设置每块为5MB part_size = 10 * 1024 * 1024 if left_size < part_size: part_size = left_size print total_size - left_size, "/", total_size response = bos_client.upload_part_from_file(bucket_name, object_key, upload_id, part_number, part_size, path, offset) left_size -= part_size offset += part_size part_list.append({ "partNumber": part_number, "eTag": response.metadata.etag }) part_number += 1 print total_size, "/", total_size bos_client.complete_multipart_upload(bucket_name, object_key, upload_id, part_list) file_name = os.path.basename(path) status_code, data = submit(object_key, token, file_name) if status_code != 200: print "服务异常,请稍候再试" data = json.loads(data) if data['errorCode'] == 0: print data['result'] else: print data['errorMsg']
class Bd_Bos_Upload: def __init__(self, bucket_name, file_name, download, withRoot, upload_class=1): self.bos_client = BosClient(bos_sample_conf.config) self.bucket_name = bucket_name self.file_name = file_name self.user_headers = { 'Content-Disposition': 'attachement; filename=' + file_name } self.download = download self.withRoot = withRoot if upload_class == 1: self._part_upload() if upload_class == 0: self._small_upload() #small file upload def _small_upload(self): data = open(self.file_name, 'rb') print('Start upload file: ', self.file_name) if self.download: user_headers = self.user_headers else: user_headers = None if self.withRoot: file_path = self.file_name else: file_path = _pickPath(self.file_name) print(self.bucket_name, file_path, self.file_name) self.bos_client.put_object_from_file(self.bucket_name, file_path, self.file_name, user_headers=user_headers) print('Upload Successfully!') #part upload , file size > 5GB def _part_upload(self): upload_id = self.bos_client.initiate_multipart_upload( self.bucket_name, self.file_name).upload_id left_size = os.path.getsize(self.file_name) offset = 0 part_number = 1 part_list = [] print('Start upload file: ', self.file_name) while left_size > 0: part_size = 5 * 1024 * 1024 if left_size < part_size: part_size = left_size response = self.bos_client.upload_part_from_file( self.bucket_name, self.file_name, upload_id, part_number, part_size, self.file_name, offset) left_size -= part_size offset += part_size part_list.append({ "partNumber": part_number, "eTag": response.metadata.etag }) print('Upload Successfully: Part', part_number) part_number += 1 self.bos_client.complete_multipart_upload(self.bucket_name, self.object_key, upload_id, part_list) print('Upload Successfully!')
class BaiduBos: def __init__(self, bucket): self._bucket = bucket self._bos_client = BosClient(_config) def upload_file(self, fn, key, get_url=False, absent=True, expiration_in_seconds=-1): """ 上传文件,如果文件超过25兆,将采用分块上传 如果key已存在,则返回key的url :param fn: :param key: :param get_url:是否需要获取key对应的url :param absent: True时,如果bos已存在该文件 则不上传 :return: """ exists = False if absent: for obj in self._bos_client.list_all_objects(self._bucket): if obj.key == key: LOGGER.warning( "the key '{0}' has already existed, upload canceled". format(key)) exists = True break if not exists: fs = os.path.getsize(fn) with open(fn, mode='rb') as f: if fs > 25 * 1024 * 1024: self._multipart_upload(fn, key) else: self._bos_client.put_object(self._bucket, key, f, fs, self.md5_file(fn)) if get_url: url = self._bos_client.generate_pre_signed_url( self._bucket, key, expiration_in_seconds=expiration_in_seconds) return url.decode("utf-8") return None def upload_bytes(self, byte_arr, key, get_url=False, absent=True, expiration_in_seconds=-1): """ 上传字节 如果key已存在,则返回key的url :param byte_arr: :param key: :return: """ exists = False if absent: for obj in self._bos_client.list_all_objects(self._bucket): if obj.key == key: LOGGER.warning( "the key '{0}' has already existed, upload canceled". format(key)) exists = True break if not exists: self._bos_client.put_object(GENIOUS_BUCKET, key, io.BytesIO(byte_arr), len(byte_arr), self.md5_obj(byte_arr)) if get_url: url = self._bos_client.generate_pre_signed_url( self._bucket, key, expiration_in_seconds=expiration_in_seconds) return url.decode("utf-8") return None def _multipart_upload(self, fn, key): """ 文件分块上传 如果key已存在,则返回key的url :arg key :arg fn """ upload_id = self._bos_client.initiate_multipart_upload( GENIOUS_BUCKET, key).upload_id left_size = os.path.getsize(fn) # left_size用于设置分块开始位置 # 设置分块的开始偏移位置 offset = 0 part_number = 1 part_list = [] index = 0 while left_size > 0: # 设置每块为5MB part_size = 5 * 1024 * 1024 if left_size < part_size: part_size = left_size response = self._bos_client.upload_part_from_file( GENIOUS_BUCKET, key, upload_id, part_number, part_size, fn, offset) index += 1 print(index) left_size -= part_size offset += part_size part_list.append({ "partNumber": part_number, "eTag": response.metadata.etag }) part_number += 1 location = self._bos_client.complete_multipart_upload( GENIOUS_BUCKET, key, upload_id, part_list) print(location.location) return location def md5_file(self, fn): buf_size = 8192 md5 = hashlib.md5() with open(fn, mode='rb') as fp: while True: bytes_to_read = buf_size buf = fp.read(bytes_to_read) if not buf: break md5.update(buf) content_md5 = base64.standard_b64encode(md5.digest()) return content_md5 def md5_obj(self, bs): md5 = hashlib.md5() md5.update(bs) return base64.standard_b64encode(md5.digest()) def list_uploaded_objects(self, prefix=None): """ 列出桶中的文件,如果提供了prefix,则最多返回1000条记录 若无法满足需要,可以使用sdk的api进行获取 :arg 指定返回key的前缀""" keys = [] if prefix is not None: response = self._bos_client.list_objects(self._bucket, prefix=prefix, max_keys=1000) for obj in response.contents: keys.append(obj.key) return keys response = self._bos_client.list_all_objects(self._bucket) for obj in response.contents: keys.append(obj.key) return keys def file_exists(self, fn): """ :arg 文件名是否存在,服务器上的文件名为key去掉前缀(带slash)后的 :return 如果文件存在,返回文件url,否则返回None """ keys = self.list_uploaded_objects() for key in keys: slash_index = key.rfind("/") if slash_index > 0: file_name = key[slash_index + 1:] if file_name == fn: url = self._bos_client.generate_pre_signed_url( bucket_name=GENIOUS_BUCKET, key=key, expiration_in_seconds=-1) return url.decode("utf-8") return None def key_exists(self, key): keys = self.list_uploaded_objects() return keys.index(key) >= 0
class UBosClient(object): """ BosClient """ def __init__(self, access_key_id, secret_access_key, bucket_name='', endpoint=''): """ 初始化 """ super(UBosClient, self).__init__() # 创建BceClientConfiguration config = BceClientConfiguration(credentials=BceCredentials( access_key_id, secret_access_key), endpoint=endpoint) # 设置请求超时时间 config.connection_timeout_in_mills = 3000 # 新建BosClient self.client = BosClient(config) self.bucket = bucket_name def check_bucket(self): """ 校验bucket是否存在 """ return not not self.client.does_bucket_exist(self.bucket) def check_object_key(self, object_key): """ 校验文件对象是否存在 """ if not self.check_bucket(): return False try: self.client.get_object_meta_data(self.bucket, object_key) return True except: return False def mkdir(self, dir_name): """ 创建文件夹 """ if not self.check_bucket(): return False try: self.client.put_object_from_string(self.bucket, '{}/'.format(dir_name), '') return True except: return False def get_all_files(self): """ 获取bucket所有文件对象集 """ file_list = [] if not self.check_bucket(): return file_list for fobj in self.client.list_all_objects(self.bucket): file_list.append({'name': fobj.key, 'size': fobj.size}) return filelist def get_files_by_dir(self, dir_name): """ 获取文件夹子文件对象集 """ file_list = [] if not self.check_bucket(): return file_list prefix = '{}/'.format(dir_name) response = self.client.list_objects(self.bucket, prefix=prefix) for fobj in response.contents: if fobj.key == prefix: continue file_list.append({'name': fobj.key, 'size': fobj.size}) return file_list def rmfile(self, object_key): """ 单一删除文件对象 """ if not self.check_bucket(): return False self.client.delete_object(self.bucket, object_key) return True def rmfiles(self, object_keys): """ 批量删除 """ if not self.check_bucket(): return False self.client.delete_multiple_objects(self.bucket, object_keys) return True def rmdir(self, dir_name): """ 删除目录, 需保证目录下无对象存在 """ if not self.check_bucket(): return False prefix = '{}/'.format(dir_name) file_list = self.get_files_by_dir(dir_name) object_keys = [fobj['name'] for fobj in file_list if 'name' in fobj] self.rmfiles(object_keys) self.client.delete_object(self.bucket, prefix) return True def single_upload(self, object_key, file_path): """ 一次性上传 """ if not self.check_bucket(): return False suffix = filename.split('.')[-1].lower() if suffix == 'mp4': ret = self.client.put_object_from_file(self.bucket, object_key, file_path, content_type='video/mp4') elif suffix in ['jpg', 'jpeg']: ret = self.client.put_object_from_file(self.bucket, object_key, file_path, content_type='image/jpeg') else: ret = self.client.put_object_from_file(self.bucket, object_key, file_path) print ret return True def get_upload_id(self, object_key): """ 断点续传获取upload_id """ upload_id = None response = self.client.list_multipart_uploads(self.bucket) for item in response.uploads: if item.key == object_key: upload_id = item.upload_id break if not upload_id: upload_id = self.client.initiate_multipart_upload( self.bucket, object_key, content_type='video/mp4').upload_id return upload_id def multipart_upload(self, object_key, file_path): """ 断点续传 """ upload_id = self.get_upload_id(object_key) if not upload_id: return False left_size = os.path.getsize(file_path) offset, part_number, part_list = 0, 1, [] while left_size > 0: # 设置每块为5MB part_size = 5 * 1024 * 1024 if left_size < part_size: part_size = left_size for _ in (_ for _ in range(300)): try: response = self.client.upload_part_from_file( self.bucket, object_key, upload_id, part_number, part_size, file_path, offset) break except: pass left_size -= part_size offset += part_size part_list.append({ "partNumber": part_number, "eTag": response.metadata.etag }) part_number += 1 for _ in (_ for _ in range(300)): try: ret = self.client.complete_multipart_upload( self.bucket, object_key, upload_id, part_list) print ret return False except: pass def upload_file(self, object_key, file_path): """ 上传文件 """ file_size = os.path.getsize(file_name) if file_size > 5 * 1024 * 1024: # 断点续传 self.multipart_upload(object_key, file_path) else: # 一次性上传 self.single_upload(object_key, file_path)