def upload_file(key, file_name):
logging.basicConfig(
level=logging.DEBUG,
format=
'%(asctime)s%(filename)s[line:%(lineno)d] %(levelname)s%(message)s',
datefmt='%a,%d %b %Y %H:%M:%S',
filename='/tmp/bos.log',
filemode='a')
__logger = logging.getLogger(__name__)
bucket_name = 'mysqlbackup'
######################################################################################################
# bucket operation samples
######################################################################################################
# create a bos client
bos_client = BosClient(bos_conf.config)
# check if bucket exists
#if not bos_client.does_bucket_exist(bucket_name):
# bos_client.create_bucket(bucket_name)
######################################################################################################
# object operation samples
######################################################################################################
# put a file as object
bos_client.put_object_from_file(bucket_name, key, file_name)
def file_upload_bos(config, local_filename):
try:
bucket_name = config['bucket_name']
bucket_name = str(bucket_name)
#p = config['dir']
AK = config['AK']
SK = config['SK']
HOST = config['HOST']
#print('step 1')
#print('file name:',local_filename)
bos_config = BceClientConfiguration(credentials=BceCredentials(
str(AK), str(SK)),
endpoint=str(HOST))
bos_client = BosClient(bos_config)
#print('step 2')
#print 'bucket_name:',bucket_name,type(bucket_name)
if not bos_client.does_bucket_exist(bucket_name):
bos_client.create_bucket(bucket_name)
#print('step 3')
key = str(local_filename.split('/')[-1])
#print('key:',key)
bos_client.put_object_from_file(bucket_name, str(key),
str(local_filename))
print(local_filename + ' has been uploaded to bucket:' + bucket_name)
return 0
except Exception as e:
print('file_upload_bos exception:' + str(e))
traceback.print_exc()
return -1
class BosConfigClient(object):
def __init__(self, bos_ak, bos_sk, bos_sts, bos_host="bj.bcebos.com"):
self.config = BceClientConfiguration(credentials=BceCredentials(
bos_ak, bos_sk),
endpoint=bos_host,
security_token=bos_sts)
self.bos_client = BosClient(self.config)
def exists(self, path):
bucket_name, object_key = get_object_info(path)
try:
self.bos_client.get_object_meta_data(bucket_name, object_key)
return True
except exception.BceError:
return False
def makedirs(self, path):
if not path.endswith('/'):
path += '/'
if self.exists(path):
return
bucket_name, object_key = get_object_info(path)
if not object_key.endswith('/'):
object_key += '/'
init_data = b''
self.bos_client.append_object(bucket_name=bucket_name,
key=object_key,
data=init_data,
content_md5=content_md5(init_data),
content_length=len(init_data))
@staticmethod
def join(path, *paths):
result = os.path.join(path, *paths)
result.replace('\\', '/')
return result
def upload_object_from_file(self, path, filename):
if not self.exists(path):
self.makedirs(path)
bucket_name, object_key = get_object_info(path)
object_key = self.join(object_key, filename)
# if not object_key.endswith('/'):
# object_key += '/'
print('Uploading file `%s`' % filename)
self.bos_client.put_object_from_file(bucket=bucket_name,
key=object_key,
file_name=filename)
__logger.debug("[Sample] list buckets:%s", bucket.name)
######################################################################################################
# object operation samples
######################################################################################################
# put a string as object
bos_client.put_object_from_string(bucket_name, key, "This is string content.")
# get a object as string
content = bos_client.get_object_as_string(bucket_name, key)
__logger.debug("[Sample] get object as string:%s", content)
# put a file as object
_create_file(file_name, 4096)
bos_client.put_object_from_file(bucket_name, key, file_name)
# get object into file
bos_client.get_object_to_file(bucket_name, key, download)
__logger.debug("[Sample] get object into file, file size:%s", os.path.getsize(download))
# copy a object
bos_client.copy_object(bucket_name, key, bucket_name, key + ".copy",)
# list objects in a bucket(up to 1000)
response = bos_client.list_objects(bucket_name)
for obj in response.contents:
__logger.debug("[Sample] list objects key:%s", obj.key)
# delete an object
bos_client.delete_object(bucket_name, key)
######################################################################################################
# object operation samples
######################################################################################################
# put a string as object
bos_client.put_object_from_string(bucket_name, key,
"This is string content.")
# get a object as string
content = bos_client.get_object_as_string(bucket_name, key)
__logger.debug("[Sample] get object as string:%s", content)
# put a file as object
_create_file(file_name, 4096)
bos_client.put_object_from_file(bucket_name, key, file_name)
# get object into file
bos_client.get_object_to_file(bucket_name, key, download)
__logger.debug("[Sample] get object into file, file size:%s",
os.path.getsize(download))
# copy a object
bos_client.copy_object(
bucket_name,
key,
bucket_name,
key + ".copy",
)
# list objects in a bucket(up to 1000)
#coding=utf-8
#导入BosClient配置文件
import bos_sample_conf
import sys
#导入BOS相关模块
from baidubce import exception
from baidubce.services import bos
from baidubce.services.bos import canned_acl
from baidubce.services.bos.bos_client import BosClient
#新建BosClient
bos_client = BosClient(bos_sample_conf.config)
path = sys.argv[1]
file_name = sys.argv[2]
bos_client.put_object_from_file("paddle-serving",
"{}/{}".format(path, file_name),
"{}".format(file_name))
print("upload {} to paddle/serving/{} success".format(file_name, path))
######################################################################################################
# object operation samples
######################################################################################################
# put a string as object
bos_client.put_object_from_string(bucket_name, key,
"This is string content.")
# get a object as string
content = bos_client.get_object_as_string(bucket_name, key)
__logger.debug("[Sample] get object as string:%s", content)
# put a file as object
_create_file(file_name, 4096)
bos_client.put_object_from_file(bucket_name, key, file_name)
# get object into file
bos_client.get_object_to_file(bucket_name, key, download)
__logger.debug("[Sample] get object into file, file size:%s",
os.path.getsize(download))
# put an appendable object
append_key = 'test_append_key'
result = bos_client.append_object_from_string(
bucket_name=bucket_name,
key=append_key,
data='This is string content.')
next_offset = result.metadata.bce_next_append_offset
bos_client.append_object_from_string(bucket_name=bucket_name,
#导入BosClient配置文件
import bos_conf
#导入BOS相关模块
from baidubce import exception
from baidubce.services import bos
from baidubce.services.bos import canned_acl
from baidubce.services.bos.bos_client import BosClient
from tqdm import tqdm
import os
#新建BosClient
bos_client = BosClient(bos_conf.config)
bucket_name = 'lits'
local_dir = '/tmp/data/preprocess/'
bos_dir = 'data/prep_raw/'
for file_name in tqdm(os.listdir(local_dir)):
# print(file_name)
bos_client.put_object_from_file(bucket_name, bos_dir + file_name, os.path.join(local_dir, file_name) )
# input('pause')
class Bd_Bos_Upload:
def __init__(self,
bucket_name,
file_name,
download,
withRoot,
upload_class=1):
self.bos_client = BosClient(bos_sample_conf.config)
self.bucket_name = bucket_name
self.file_name = file_name
self.user_headers = {
'Content-Disposition': 'attachement; filename=' + file_name
}
self.download = download
self.withRoot = withRoot
if upload_class == 1:
self._part_upload()
if upload_class == 0:
self._small_upload()
#small file upload
def _small_upload(self):
data = open(self.file_name, 'rb')
print('Start upload file: ', self.file_name)
if self.download:
user_headers = self.user_headers
else:
user_headers = None
if self.withRoot:
file_path = self.file_name
else:
file_path = _pickPath(self.file_name)
print(self.bucket_name, file_path, self.file_name)
self.bos_client.put_object_from_file(self.bucket_name,
file_path,
self.file_name,
user_headers=user_headers)
print('Upload Successfully!')
#part upload , file size > 5GB
def _part_upload(self):
upload_id = self.bos_client.initiate_multipart_upload(
self.bucket_name, self.file_name).upload_id
left_size = os.path.getsize(self.file_name)
offset = 0
part_number = 1
part_list = []
print('Start upload file: ', self.file_name)
while left_size > 0:
part_size = 5 * 1024 * 1024
if left_size < part_size:
part_size = left_size
response = self.bos_client.upload_part_from_file(
self.bucket_name, self.file_name, upload_id, part_number,
part_size, self.file_name, offset)
left_size -= part_size
offset += part_size
part_list.append({
"partNumber": part_number,
"eTag": response.metadata.etag
})
print('Upload Successfully: Part', part_number)
part_number += 1
self.bos_client.complete_multipart_upload(self.bucket_name,
self.object_key, upload_id,
part_list)
print('Upload Successfully!')
class UBosClient(object):
"""
BosClient
"""
def __init__(self,
access_key_id,
secret_access_key,
bucket_name='',
endpoint=''):
"""
初始化
"""
super(UBosClient, self).__init__()
# 创建BceClientConfiguration
config = BceClientConfiguration(credentials=BceCredentials(
access_key_id, secret_access_key),
endpoint=endpoint)
# 设置请求超时时间
config.connection_timeout_in_mills = 3000
# 新建BosClient
self.client = BosClient(config)
self.bucket = bucket_name
def check_bucket(self):
"""
校验bucket是否存在
"""
return not not self.client.does_bucket_exist(self.bucket)
def check_object_key(self, object_key):
"""
校验文件对象是否存在
"""
if not self.check_bucket():
return False
try:
self.client.get_object_meta_data(self.bucket, object_key)
return True
except:
return False
def mkdir(self, dir_name):
"""
创建文件夹
"""
if not self.check_bucket():
return False
try:
self.client.put_object_from_string(self.bucket,
'{}/'.format(dir_name), '')
return True
except:
return False
def get_all_files(self):
"""
获取bucket所有文件对象集
"""
file_list = []
if not self.check_bucket():
return file_list
for fobj in self.client.list_all_objects(self.bucket):
file_list.append({'name': fobj.key, 'size': fobj.size})
return filelist
def get_files_by_dir(self, dir_name):
"""
获取文件夹子文件对象集
"""
file_list = []
if not self.check_bucket():
return file_list
prefix = '{}/'.format(dir_name)
response = self.client.list_objects(self.bucket, prefix=prefix)
for fobj in response.contents:
if fobj.key == prefix:
continue
file_list.append({'name': fobj.key, 'size': fobj.size})
return file_list
def rmfile(self, object_key):
"""
单一删除文件对象
"""
if not self.check_bucket():
return False
self.client.delete_object(self.bucket, object_key)
return True
def rmfiles(self, object_keys):
"""
批量删除
"""
if not self.check_bucket():
return False
self.client.delete_multiple_objects(self.bucket, object_keys)
return True
def rmdir(self, dir_name):
"""
删除目录, 需保证目录下无对象存在
"""
if not self.check_bucket():
return False
prefix = '{}/'.format(dir_name)
file_list = self.get_files_by_dir(dir_name)
object_keys = [fobj['name'] for fobj in file_list if 'name' in fobj]
self.rmfiles(object_keys)
self.client.delete_object(self.bucket, prefix)
return True
def single_upload(self, object_key, file_path):
"""
一次性上传
"""
if not self.check_bucket():
return False
suffix = filename.split('.')[-1].lower()
if suffix == 'mp4':
ret = self.client.put_object_from_file(self.bucket,
object_key,
file_path,
content_type='video/mp4')
elif suffix in ['jpg', 'jpeg']:
ret = self.client.put_object_from_file(self.bucket,
object_key,
file_path,
content_type='image/jpeg')
else:
ret = self.client.put_object_from_file(self.bucket, object_key,
file_path)
print ret
return True
def get_upload_id(self, object_key):
"""
断点续传获取upload_id
"""
upload_id = None
response = self.client.list_multipart_uploads(self.bucket)
for item in response.uploads:
if item.key == object_key:
upload_id = item.upload_id
break
if not upload_id:
upload_id = self.client.initiate_multipart_upload(
self.bucket, object_key, content_type='video/mp4').upload_id
return upload_id
def multipart_upload(self, object_key, file_path):
"""
断点续传
"""
upload_id = self.get_upload_id(object_key)
if not upload_id:
return False
left_size = os.path.getsize(file_path)
offset, part_number, part_list = 0, 1, []
while left_size > 0:
# 设置每块为5MB
part_size = 5 * 1024 * 1024
if left_size < part_size:
part_size = left_size
for _ in (_ for _ in range(300)):
try:
response = self.client.upload_part_from_file(
self.bucket, object_key, upload_id, part_number,
part_size, file_path, offset)
break
except:
pass
left_size -= part_size
offset += part_size
part_list.append({
"partNumber": part_number,
"eTag": response.metadata.etag
})
part_number += 1
for _ in (_ for _ in range(300)):
try:
ret = self.client.complete_multipart_upload(
self.bucket, object_key, upload_id, part_list)
print ret
return False
except:
pass
def upload_file(self, object_key, file_path):
"""
上传文件
"""
file_size = os.path.getsize(file_name)
if file_size > 5 * 1024 * 1024:
# 断点续传
self.multipart_upload(object_key, file_path)
else:
# 一次性上传
self.single_upload(object_key, file_path)