def test_bucket_iterator(self):
service = oss2.Service(oss2.Auth(OSS_ID, OSS_SECRET), OSS_ENDPOINT)
self.assertTrue(
OSS_BUCKET in (b.name
for b in oss2.BucketIterator(service, max_keys=2)))
self.assertTrue(OSS_BUCKET in list(
b.name for b in oss2.BucketIterator(service, max_keys=2)))
def showBucket():
print("********** 获取bucket信息 *******")
service = oss2.Service(oss2.Auth(access_key_id, access_key_secret),
endpoint)
print("*****************************")
print(" 现有bucket有： ")
print('\n'.join(info.name for info in oss2.BucketIterator(service)))
buckets = oss2.BucketIterator(service)
print("*****************************")
def oss(ak, sk, region):
id_list = []
auth = oss2.Auth(ak, sk)
service = oss2.Service(auth, 'http://oss-cn-hangzhou.aliyuncs.com')
for b in oss2.BucketIterator(service):
id_list.append({"l": "", "d": b.name})
return id_list
def oss_info(self) -> GaugeMetricFamily:
auth = oss2.Auth(self.ak, self.secret)
service = oss2.Service(
auth, 'http://oss-{resion_id}.aliyuncs.com'.format(
resion_id=self.region_id))
nested_handler = None
gauge = None
label_keys = None
for instance in oss2.BucketIterator(service):
bucket = oss2.Bucket(auth, 'http://oss-cn-beijing.aliyuncs.com',
instance.name)
bucket_info = bucket.get_bucket_info()
instance_dict = {
'name': bucket_info.name,
'storage_class': bucket_info.storage_class,
'creation_date': bucket_info.creation_date,
'intranet_endpoint': bucket_info.intranet_endpoint,
'extranet_endpoint': bucket_info.extranet_endpoint,
'owner': bucket_info.owner.id,
'grant': bucket_info.acl.grant,
'data_redundancy_type': bucket_info.data_redundancy_type,
}
if gauge == None:
label_keys = self.label_keys(instance_dict, nested_handler)
gauge = GaugeMetricFamily('aliyun_meta_oss_info',
'',
labels=label_keys)
gauge.add_metric(labels=self.label_values(instance_dict,
label_keys,
nested_handler),
value=1.0)
return gauge
def selectbucketlist(self):
"""查询bucket列表"""
buckets_list = []
for i in oss2.BucketIterator(self.service):
buckets_list.append(i.name)
#return buckets_list
print buckets_list
def createBucket():
""" 创建bucket """
auth = oss2.Auth(accessKeyId, accessKeySecret)
service = oss2.Service(auth, endpoint)
print[b.name for b in oss2.BucketIterator(service)]
bucket = oss2.Bucket(auth, endpoint, bucket_name)
bucket.create_bucket(oss2.models.BUCKET_ACL_PRIVATE)
return bucket
def showBucket(self):
print("********** 获取bucket信息 *******")
service = oss2.Service(
oss2.Auth(self.access_key_id, self.access_key_secret),
self.endpoint)
print("*****************************")
print(" 现有bucket有: ")
print('\n'.join(info.name for info in oss2.BucketIterator(service)))
print("*****************************")
def list_buckets(self):
r = None
e = None
try:
service = oss2.Service(self._auth, self.endpoint)
# r = oss2.BucketIterator(service)
r = [b.name for b in oss2.BucketIterator(service)]
except oss2.exceptions.ServerError as err:
e = json.loads(str(err).replace("'", '"'))
log.debug(r or e)
return r, e
def get_oss_list(self):
error_dict = {}
try:
self.oss_list_obj = [b for b in oss2.BucketIterator(self.service)]
self.oss_list = []
for i in self.oss_list_obj:
print(i)
self.oss_list.append(i.__dict__)
return self.oss_list
except Exception as e:
error_dict.update(e.__dict__)
return error_dict
def list_buckets(self):
service = oss2.Service(auth, url)
service = oss2.BucketIterator(service)
lst = []
for x in service:
obj = {}
obj['name'] = x.name
obj['storage_class'] = x.storage_class
obj['creation_date'] = x.creation_date
obj['intranet_endpoint'] = x.intranet_endpoint
obj['extranet_endpoint'] = x.extranet_endpoint
lst.append(obj)
return lst
def iter_buckets(self, prefix='', marker='', max_keys=100, max_retries=None):
"""
:param prefix: 只列举匹配该前缀的Bucket
:param marker: 分页符。只列举Bucket名字典序在此之后的Bucket
:param max_keys: 每次调用 `list_buckets` 时的max_keys参数。注意迭代器返回的数目可能会大于该值。
:param max_retries:
:return:
"""
if not hasattr(self, 'service'):
self.service = oss2.Service(self.auth, self.endpoint)
return oss2.BucketIterator(
self.service, prefix=prefix, marker=marker,
max_keys=max_keys, max_retries=max_retries)
def list_buckets(self, **kwargs):
resp = {'status': 'success', 'errmsg': '', 'data': []}
try:
service = oss2.Service(self.auth, self.endpoint)
resp['data'] = ([{
'name':
b.name,
'create_time':
datetime.datetime.fromtimestamp(b.creation_date),
'location':
b.location
} for b in oss2.BucketIterator(service)])
except Exception as e:
resp = {'status': 'failure', 'errmsg': str(e)}
return resp
def __init__(self,
access_key_id,
access_key_secret,
host,
bucket_name,
keeps=None):
"""
access_key_id:
access_key_secret:
host: 域名，如 http://oss-cn-hangzhou.aliyuncs.com
bucket_name: 不需要在后台自动创建，也会自动创建好
"""
super(OSSBackend, self).__init__(keeps)
auth = oss2.Auth(access_key_id, access_key_secret)
self.bucket = oss2.Bucket(auth, host, bucket_name)
service = oss2.Service(auth, host)
if bucket_name not in [b.name for b in oss2.BucketIterator(service)]:
# 说明bucket不存在
self.bucket.create_bucket()
def _lsbuckets(self, refresh=False):
if "" not in self.dircache or refresh:
if self.anon:
# cannot list buckets if not logged in
return []
try:
service = oss2.Service(auth, self.endpoint)
files = [b.name for b in oss2.BucketIterator(service)]
except Exception as e:
# list bucket permission missing
raise e
for f in files:
f["Key"] = f["Name"]
f["Size"] = 0
f["StorageClass"] = "BUCKET"
f["size"] = 0
f["type"] = "directory"
f["name"] = f["Name"]
del f["Name"]
self.dircache[""] = files
return files
return self.dircache[""]
def create_space(bucket,
service,
name=None,
delete=False): #创建一个桶，用来存放数据，如果桶已经存在，则不创建
print([b.name for b in oss2.BucketIterator(service)]) # 列出所有存在的桶
def read_object(delete_obj=False): # oss2.ObjectIteratorr用于遍历文件。遍历桶内每个文件
for b in islice(oss2.ObjectIterator(bucket), 1, None):
print(b.key)
if delete_obj:
bucket.delete_object(b.key)
print("delete %s success" % b.key)
def does_bucket_exist(bucket): #判断存储空间是否存在
try:
bucket.get_bucket_info()
except oss2.exceptions.NoSuchBucket:
return False
except:
raise
return True
if delete:
try:
# 删除存储空间。
bucket.delete_bucket()
except oss2.exceptions.BucketNotEmpty:
print('bucket is not empty.')
except oss2.exceptions.NoSuchBucket:
print('bucket does not exist')
else:
if does_bucket_exist(bucket): # 如果存在此桶，则不创建存储空间
print("sorry bucket %s has exist." % name)
else: # 不存在则创建
bucket.create_bucket()
print("creat bucket %s success." % name)
def create_space(name=None,delete=False): #创建云端的桶，用来存储数据
# 阿里云主账号AccessKey拥有所有API的访问权限，风险很高。强烈建议您创建并使用RAM账号进行API访问或日常运维，请登录 https://ram.console.aliyun.com 创建RAM账号。
auth = oss2.Auth('<yourAccessKeyId>', '<yourAccessKeySecret>')
# 通过指定Endpoint和存储空间名称，您可以在指定的地域创建新的存储空间。Endpoint以杭州为例，其它Region请按实际情况填写。
bucket = oss2.Bucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', '<yourBucketName>')
service = oss2.Service(auth, 'http://oss-cn-beijing.aliyuncs.com') #查看存储空间
print([b.name for b in oss2.BucketIterator(service)]) # 列出所有存在的桶
def read_object(delete_obj=False): # oss2.ObjectIteratorr用于遍历文件。遍历桶内每个文件
for b in islice(oss2.ObjectIterator(bucket), 1, None):
print(b.key)
if delete_obj:
bucket.delete_object(b.key)
print("delete %s success" % b.key)
def does_bucket_exist(bucket):#判断存储空间是否存在
try:
bucket.get_bucket_info()
except oss2.exceptions.NoSuchBucket:
return False
except:
raise
return True
if delete:
try:
# 删除存储空间。
bucket.delete_bucket()
except oss2.exceptions.BucketNotEmpty:
print('bucket is not empty.')
except oss2.exceptions.NoSuchBucket:
print('bucket does not exist')
else:
if does_bucket_exist(bucket): # 如果存在此桶，则不创建存储空间
print("sorry bucket %s has exist." % name)
else: # 不存在则创建
bucket.create_bucket()
print("creat bucket %s success." % name)
def reset(self):
for i in range(RETRY_COUNT):
try:
buckets = oss2.BucketIterator(self.service)
if not buckets:
break
for bucket in buckets:
try:
bucket = oss2.Bucket(self.auth, self.region,
str(bucket.name))
for obj in islice(oss2.ObjectIterator(bucket), 10):
bucket.delete_object(obj.key)
try:
bucket.delete_bucket()
except oss2.exceptions.BucketNotEmpty:
print('bucket is not empty.')
except oss2.exceptions.NoSuchBucket:
print('bucket does not exist')
except oss2.exceptions.ClientError:
pass
except oss2.exceptions.ClientError:
pass
def listAllBuckets(self):
return [b.name for b in oss2.BucketIterator(self.service)]
# 通过环境变量获取，或者把诸如“<你的AccessKeyId>”替换成真实的AccessKeyId等。
access_key_id = os.getenv('OSS_TEST_ACCESS_KEY_ID', '<你的AccessKeyId>')
access_key_secret = os.getenv('OSS_TEST_ACCESS_KEY_SECRET',
'<你的AccessKeySecret>')
bucket_name = os.getenv('OSS_TEST_BUCKET', '<你的Bucket>')
endpoint = os.getenv('OSS_TEST_ENDPOINT', '<你的访问域名>')
# 确认上面的参数都填写正确了
for param in (access_key_id, access_key_secret, bucket_name, endpoint):
assert '<' not in param, '请设置参数：' + param
# 列举所有的Bucket
# 1. 先创建一个Service对象
# 2. 用oss2.BucketIterator遍历
service = oss2.Service(oss2.Auth(access_key_id, access_key_secret), endpoint)
print('\n'.join(info.name for info in oss2.BucketIterator(service)))
# 创建Bucket对象，所有Object相关的接口都可以通过Bucket对象来进行
bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint,
bucket_name)
# 下面只展示如何配置静态网站托管。其他的Bucket操作方式类似，可以参考tests/test_bucket.py里的内容
# 方法一：可以生成一个BucketWebsite对象来设置
bucket.put_bucket_website(oss2.models.BucketWebsite('index.html',
'error.html'))
# 方法二：可以直接设置XML
xml = '''
<WebsiteConfiguration>
<IndexDocument>
def list_buckets(self, **kwargs):
raw_buckets = oss2.BucketIterator(self.service)
buckets = [{'id': b.name} for b in raw_buckets]
return buckets
def getAllBucket():
service = oss2.Service(
oss2.Auth('LTAI4Fm1MACEyXSzXMW6PT1m',
'rCIsBsNYc9nxL2cxhx4PJp2V6FBAsP'),
'http://oss-cn-hongkong.aliyuncs.com')
print('\n'.join(info.name for info in oss2.BucketIterator(service)))
# -*- coding: utf-8 -*-
import oss2
import time
auth = oss2.Auth('LTAIeEscH1ag5dWj', '1R6ukC8o98clQxNsLDCPI3Is0t4003')
service = oss2.Service(auth, 'oss-cn-huhehaote.aliyuncs.com')
print([b.name for b in oss2.BucketIterator(service)])
bucket = oss2.Bucket(auth, 'oss-cn-huhehaote.aliyuncs.com', 'apk2ali')
from itertools import islice
print(time.strftime('%Y-%m-%d %H:%M:%S'))
# for b in islice(oss2.ObjectIterator(bucket), 10):
# print(b.key)
# bucket.delete_object(b.key)
# print time.strftime('%Y-%m-%d %H:%M:%S')
#
for b in islice(oss2.ObjectIterator(bucket), 10):
print(b.key)
#
print(time.strftime('%Y-%m-%d %H:%M:%S'))
# bucket.put_object_from_file(
# 'qq.apk',
# '/Users/lvjinzhao/Documents/onlineAutoTest/dailyTest/pageGet/media/appfile/dFSJ3D-release_2017_09_04_1843正式_205_5.0.3.apk'
# )
print(time.strftime('%Y-%m-%d %H:%M:%S'))
bucket.get_object_to_file('qq.apk', 'local-backup2.apk')
print(time.strftime('%Y-%m-%d %H:%M:%S'))
def list_buckets(self):
service = oss2.Service(self.auth, self.endpoint)
return ([b.name for b in oss2.BucketIterator(service)])
def getBuckets(self):
return oss2.BucketIterator(self.service)
def list_bucket(self):
return oss2.BucketIterator(self.service)
def hello(accessKey, secret):
auth = oss2.Auth(accessKey, secret)
service = oss2.Service(auth, 'http://oss-cn-beijing.aliyuncs.com')
return json.dumps([b.name for b in oss2.BucketIterator(service)])
def get_buckets(self):
service = oss2.Service(self.auth, self.conf.oss_endpoint)
# 查看Bucket列表
print([b.name for b in oss2.BucketIterator(service)])
def get_all_buckets_name():
logger.info('--------获取所有【bucket name】--------')
service = oss2.Service(auth, OSS_HOST)
return [info.name for info in oss2.BucketIterator(service)]
def list_bucket(auth,endpoint):
service = oss2.Service(auth, endpoint)
print([b.name for b in oss2.BucketIterator(service)])
return True
def oss_info(self) -> GaugeMetricFamily:
auth = oss2.Auth(self.ak, self.secret)
service = oss2.Service(
auth, 'http://oss-{resion_id}.aliyuncs.com'.format(
resion_id=self.region_id))
nested_handler = None
gauge = None
label_keys = None
try:
oss_bucket_iterator = oss2.BucketIterator(service, max_retries=2)
except Exception as e:
try:
print('connect timeout, now retry......')
time.sleep(5)
oss_bucket_iterator = oss2.BucketIterator(service,
max_retries=2)
except Exception as e:
print('oss bucket iterator err')
print(e)
return GaugeMetricFamily('aliyun_meta_oss_info', '')
for instance in oss_bucket_iterator:
try:
bucket = oss2.Bucket(auth,
'http://oss-cn-beijing.aliyuncs.com',
instance.name,
connect_timeout=10)
bucket_info = bucket.get_bucket_info()
instance_dict = {
'name': bucket_info.name,
'storage_class': bucket_info.storage_class,
'creation_date': bucket_info.creation_date,
'intranet_endpoint': bucket_info.intranet_endpoint,
'extranet_endpoint': bucket_info.extranet_endpoint,
'owner': bucket_info.owner.id,
'grant': bucket_info.acl.grant,
'data_redundancy_type': bucket_info.data_redundancy_type,
}
except Exception as e:
try:
print('connect timeout, now retry...')
time.sleep(2)
bucket = oss2.Bucket(auth,
'http://oss-cn-beijing.aliyuncs.com',
instance.name,
connect_timeout=10)
bucket_info = bucket.get_bucket_info()
instance_dict = {
'name': bucket_info.name,
'storage_class': bucket_info.storage_class,
'creation_date': bucket_info.creation_date,
'intranet_endpoint': bucket_info.intranet_endpoint,
'extranet_endpoint': bucket_info.extranet_endpoint,
'owner': bucket_info.owner.id,
'grant': bucket_info.acl.grant,
'data_redundancy_type':
bucket_info.data_redundancy_type,
}
except Exception as e:
print('err key')
print(e)
continue
if gauge == None:
label_keys = self.label_keys(instance_dict, nested_handler)
gauge = GaugeMetricFamily('aliyun_meta_oss_info',
'',
labels=label_keys)
gauge.add_metric(labels=self.label_values(instance_dict,
label_keys,
nested_handler),
value=1.0)
return gauge
