def upload_file(key, file_name): logging.basicConfig( level=logging.DEBUG, format= '%(asctime)s%(filename)s[line:%(lineno)d] %(levelname)s%(message)s', datefmt='%a,%d %b %Y %H:%M:%S', filename='/tmp/bos.log', filemode='a') __logger = logging.getLogger(__name__) bucket_name = 'mysqlbackup' ###################################################################################################### # bucket operation samples ###################################################################################################### # create a bos client bos_client = BosClient(bos_conf.config) # check if bucket exists #if not bos_client.does_bucket_exist(bucket_name): # bos_client.create_bucket(bucket_name) ###################################################################################################### # object operation samples ###################################################################################################### # put a file as object bos_client.put_object_from_file(bucket_name, key, file_name)
def file_upload_bos(config, local_filename): try: bucket_name = config['bucket_name'] bucket_name = str(bucket_name) #p = config['dir'] AK = config['AK'] SK = config['SK'] HOST = config['HOST'] #print('step 1') #print('file name:',local_filename) bos_config = BceClientConfiguration(credentials=BceCredentials( str(AK), str(SK)), endpoint=str(HOST)) bos_client = BosClient(bos_config) #print('step 2') #print 'bucket_name:',bucket_name,type(bucket_name) if not bos_client.does_bucket_exist(bucket_name): bos_client.create_bucket(bucket_name) #print('step 3') key = str(local_filename.split('/')[-1]) #print('key:',key) bos_client.put_object_from_file(bucket_name, str(key), str(local_filename)) print(local_filename + ' has been uploaded to bucket:' + bucket_name) return 0 except Exception as e: print('file_upload_bos exception:' + str(e)) traceback.print_exc() return -1
class BosConfigClient(object): def __init__(self, bos_ak, bos_sk, bos_sts, bos_host="bj.bcebos.com"): self.config = BceClientConfiguration(credentials=BceCredentials( bos_ak, bos_sk), endpoint=bos_host, security_token=bos_sts) self.bos_client = BosClient(self.config) def exists(self, path): bucket_name, object_key = get_object_info(path) try: self.bos_client.get_object_meta_data(bucket_name, object_key) return True except exception.BceError: return False def makedirs(self, path): if not path.endswith('/'): path += '/' if self.exists(path): return bucket_name, object_key = get_object_info(path) if not object_key.endswith('/'): object_key += '/' init_data = b'' self.bos_client.append_object(bucket_name=bucket_name, key=object_key, data=init_data, content_md5=content_md5(init_data), content_length=len(init_data)) @staticmethod def join(path, *paths): result = os.path.join(path, *paths) result.replace('\\', '/') return result def upload_object_from_file(self, path, filename): if not self.exists(path): self.makedirs(path) bucket_name, object_key = get_object_info(path) object_key = self.join(object_key, filename) # if not object_key.endswith('/'): # object_key += '/' print('Uploading file `%s`' % filename) self.bos_client.put_object_from_file(bucket=bucket_name, key=object_key, file_name=filename)
__logger.debug("[Sample] list buckets:%s", bucket.name) ###################################################################################################### # object operation samples ###################################################################################################### # put a string as object bos_client.put_object_from_string(bucket_name, key, "This is string content.") # get a object as string content = bos_client.get_object_as_string(bucket_name, key) __logger.debug("[Sample] get object as string:%s", content) # put a file as object _create_file(file_name, 4096) bos_client.put_object_from_file(bucket_name, key, file_name) # get object into file bos_client.get_object_to_file(bucket_name, key, download) __logger.debug("[Sample] get object into file, file size:%s", os.path.getsize(download)) # copy a object bos_client.copy_object(bucket_name, key, bucket_name, key + ".copy",) # list objects in a bucket(up to 1000) response = bos_client.list_objects(bucket_name) for obj in response.contents: __logger.debug("[Sample] list objects key:%s", obj.key) # delete an object bos_client.delete_object(bucket_name, key)
###################################################################################################### # object operation samples ###################################################################################################### # put a string as object bos_client.put_object_from_string(bucket_name, key, "This is string content.") # get a object as string content = bos_client.get_object_as_string(bucket_name, key) __logger.debug("[Sample] get object as string:%s", content) # put a file as object _create_file(file_name, 4096) bos_client.put_object_from_file(bucket_name, key, file_name) # get object into file bos_client.get_object_to_file(bucket_name, key, download) __logger.debug("[Sample] get object into file, file size:%s", os.path.getsize(download)) # copy a object bos_client.copy_object( bucket_name, key, bucket_name, key + ".copy", ) # list objects in a bucket(up to 1000)
#coding=utf-8 #导入BosClient配置文件 import bos_sample_conf import sys #导入BOS相关模块 from baidubce import exception from baidubce.services import bos from baidubce.services.bos import canned_acl from baidubce.services.bos.bos_client import BosClient #新建BosClient bos_client = BosClient(bos_sample_conf.config) path = sys.argv[1] file_name = sys.argv[2] bos_client.put_object_from_file("paddle-serving", "{}/{}".format(path, file_name), "{}".format(file_name)) print("upload {} to paddle/serving/{} success".format(file_name, path))
###################################################################################################### # object operation samples ###################################################################################################### # put a string as object bos_client.put_object_from_string(bucket_name, key, "This is string content.") # get a object as string content = bos_client.get_object_as_string(bucket_name, key) __logger.debug("[Sample] get object as string:%s", content) # put a file as object _create_file(file_name, 4096) bos_client.put_object_from_file(bucket_name, key, file_name) # get object into file bos_client.get_object_to_file(bucket_name, key, download) __logger.debug("[Sample] get object into file, file size:%s", os.path.getsize(download)) # put an appendable object append_key = 'test_append_key' result = bos_client.append_object_from_string( bucket_name=bucket_name, key=append_key, data='This is string content.') next_offset = result.metadata.bce_next_append_offset bos_client.append_object_from_string(bucket_name=bucket_name,
#导入BosClient配置文件 import bos_conf #导入BOS相关模块 from baidubce import exception from baidubce.services import bos from baidubce.services.bos import canned_acl from baidubce.services.bos.bos_client import BosClient from tqdm import tqdm import os #新建BosClient bos_client = BosClient(bos_conf.config) bucket_name = 'lits' local_dir = '/tmp/data/preprocess/' bos_dir = 'data/prep_raw/' for file_name in tqdm(os.listdir(local_dir)): # print(file_name) bos_client.put_object_from_file(bucket_name, bos_dir + file_name, os.path.join(local_dir, file_name) ) # input('pause')
class Bd_Bos_Upload: def __init__(self, bucket_name, file_name, download, withRoot, upload_class=1): self.bos_client = BosClient(bos_sample_conf.config) self.bucket_name = bucket_name self.file_name = file_name self.user_headers = { 'Content-Disposition': 'attachement; filename=' + file_name } self.download = download self.withRoot = withRoot if upload_class == 1: self._part_upload() if upload_class == 0: self._small_upload() #small file upload def _small_upload(self): data = open(self.file_name, 'rb') print('Start upload file: ', self.file_name) if self.download: user_headers = self.user_headers else: user_headers = None if self.withRoot: file_path = self.file_name else: file_path = _pickPath(self.file_name) print(self.bucket_name, file_path, self.file_name) self.bos_client.put_object_from_file(self.bucket_name, file_path, self.file_name, user_headers=user_headers) print('Upload Successfully!') #part upload , file size > 5GB def _part_upload(self): upload_id = self.bos_client.initiate_multipart_upload( self.bucket_name, self.file_name).upload_id left_size = os.path.getsize(self.file_name) offset = 0 part_number = 1 part_list = [] print('Start upload file: ', self.file_name) while left_size > 0: part_size = 5 * 1024 * 1024 if left_size < part_size: part_size = left_size response = self.bos_client.upload_part_from_file( self.bucket_name, self.file_name, upload_id, part_number, part_size, self.file_name, offset) left_size -= part_size offset += part_size part_list.append({ "partNumber": part_number, "eTag": response.metadata.etag }) print('Upload Successfully: Part', part_number) part_number += 1 self.bos_client.complete_multipart_upload(self.bucket_name, self.object_key, upload_id, part_list) print('Upload Successfully!')
class UBosClient(object): """ BosClient """ def __init__(self, access_key_id, secret_access_key, bucket_name='', endpoint=''): """ 初始化 """ super(UBosClient, self).__init__() # 创建BceClientConfiguration config = BceClientConfiguration(credentials=BceCredentials( access_key_id, secret_access_key), endpoint=endpoint) # 设置请求超时时间 config.connection_timeout_in_mills = 3000 # 新建BosClient self.client = BosClient(config) self.bucket = bucket_name def check_bucket(self): """ 校验bucket是否存在 """ return not not self.client.does_bucket_exist(self.bucket) def check_object_key(self, object_key): """ 校验文件对象是否存在 """ if not self.check_bucket(): return False try: self.client.get_object_meta_data(self.bucket, object_key) return True except: return False def mkdir(self, dir_name): """ 创建文件夹 """ if not self.check_bucket(): return False try: self.client.put_object_from_string(self.bucket, '{}/'.format(dir_name), '') return True except: return False def get_all_files(self): """ 获取bucket所有文件对象集 """ file_list = [] if not self.check_bucket(): return file_list for fobj in self.client.list_all_objects(self.bucket): file_list.append({'name': fobj.key, 'size': fobj.size}) return filelist def get_files_by_dir(self, dir_name): """ 获取文件夹子文件对象集 """ file_list = [] if not self.check_bucket(): return file_list prefix = '{}/'.format(dir_name) response = self.client.list_objects(self.bucket, prefix=prefix) for fobj in response.contents: if fobj.key == prefix: continue file_list.append({'name': fobj.key, 'size': fobj.size}) return file_list def rmfile(self, object_key): """ 单一删除文件对象 """ if not self.check_bucket(): return False self.client.delete_object(self.bucket, object_key) return True def rmfiles(self, object_keys): """ 批量删除 """ if not self.check_bucket(): return False self.client.delete_multiple_objects(self.bucket, object_keys) return True def rmdir(self, dir_name): """ 删除目录, 需保证目录下无对象存在 """ if not self.check_bucket(): return False prefix = '{}/'.format(dir_name) file_list = self.get_files_by_dir(dir_name) object_keys = [fobj['name'] for fobj in file_list if 'name' in fobj] self.rmfiles(object_keys) self.client.delete_object(self.bucket, prefix) return True def single_upload(self, object_key, file_path): """ 一次性上传 """ if not self.check_bucket(): return False suffix = filename.split('.')[-1].lower() if suffix == 'mp4': ret = self.client.put_object_from_file(self.bucket, object_key, file_path, content_type='video/mp4') elif suffix in ['jpg', 'jpeg']: ret = self.client.put_object_from_file(self.bucket, object_key, file_path, content_type='image/jpeg') else: ret = self.client.put_object_from_file(self.bucket, object_key, file_path) print ret return True def get_upload_id(self, object_key): """ 断点续传获取upload_id """ upload_id = None response = self.client.list_multipart_uploads(self.bucket) for item in response.uploads: if item.key == object_key: upload_id = item.upload_id break if not upload_id: upload_id = self.client.initiate_multipart_upload( self.bucket, object_key, content_type='video/mp4').upload_id return upload_id def multipart_upload(self, object_key, file_path): """ 断点续传 """ upload_id = self.get_upload_id(object_key) if not upload_id: return False left_size = os.path.getsize(file_path) offset, part_number, part_list = 0, 1, [] while left_size > 0: # 设置每块为5MB part_size = 5 * 1024 * 1024 if left_size < part_size: part_size = left_size for _ in (_ for _ in range(300)): try: response = self.client.upload_part_from_file( self.bucket, object_key, upload_id, part_number, part_size, file_path, offset) break except: pass left_size -= part_size offset += part_size part_list.append({ "partNumber": part_number, "eTag": response.metadata.etag }) part_number += 1 for _ in (_ for _ in range(300)): try: ret = self.client.complete_multipart_upload( self.bucket, object_key, upload_id, part_list) print ret return False except: pass def upload_file(self, object_key, file_path): """ 上传文件 """ file_size = os.path.getsize(file_name) if file_size > 5 * 1024 * 1024: # 断点续传 self.multipart_upload(object_key, file_path) else: # 一次性上传 self.single_upload(object_key, file_path)