def get_users_stat(s3_servers):
users_stat = {}
try:
rgwAdmin = RGWAdmin(settings.S3_ACCESS,
settings.S3_SECRET,
s3_servers.pop(0),
secure=False)
buckets_list = rgwAdmin.get_bucket()
for bucket in buckets_list:
try:
bucket_stat = rgwAdmin.get_bucket(bucket)
if bucket_stat["owner"] in users_stat:
if "rgw.main" in bucket_stat["usage"]:
users_stat[bucket_stat["owner"]][bucket] = bucket_stat[
"usage"]["rgw.main"]
else:
users_stat[bucket_stat["owner"]][bucket] = {}
else:
if "rgw.main" in bucket_stat["usage"]:
users_stat[bucket_stat["owner"]] = {
bucket: bucket_stat["usage"]["rgw.main"]
}
else:
users_stat[bucket_stat["owner"]] = {bucket: {}}
except:
pass
return users_stat
except IndexError:
return users_stat
except TypeError:
return get_users_stat(s3_servers)
except exceptions.ServerDown:
return get_users_stat(s3_servers)
def get_users_stat(s3_server):
users_stat = defaultdict(lambda: [0, defaultdict(lambda: defaultdict(dict))])
try:
rgwAdmin = RGWAdmin(settings.S3_CRED['access_key'],
settings.S3_CRED['secret_key'],
s3_server, secure=False)
buckets_list = rgwAdmin.get_bucket()
for bucket in buckets_list:
try:
bucket_stat = rgwAdmin.get_bucket(bucket)
username = rgwAdmin.get_user(
bucket_stat["owner"]
)["display_name"]
if ':' in username:
group_name = username.split(":")[0].upper()
else:
group_name = "_"
users_stat[group_name][1][bucket_stat["owner"]][bucket] = (
bucket_stat["usage"]["rgw.main"]
)
users_stat[group_name][0] += bucket_stat[
"usage"
]["rgw.main"]["size_kb"]
except:
pass
for i in users_stat:
users_stat[i][0] *= 1024
for j in users_stat[i][1]:
users_stat[i][1][j].default_factory = None
users_stat[i][1].default_factory = None
return dict(users_stat)
except:
return dict(users_stat)
def get_users_stat(s3_servers):
users_stat = {}
try:
rgwAdmin = RGWAdmin(settings.S3_ACCESS, settings.S3_SECRET,
s3_servers.pop(0), secure=False)
buckets_list = rgwAdmin.get_bucket()
for bucket in buckets_list:
try:
bucket_stat = rgwAdmin.get_bucket(bucket)
if bucket_stat["owner"] in users_stat:
if "rgw.main" in bucket_stat["usage"]:
users_stat[
bucket_stat["owner"]
][bucket] = bucket_stat["usage"]["rgw.main"]
else:
users_stat[
bucket_stat["owner"]
][bucket] = {}
else:
if "rgw.main" in bucket_stat["usage"]:
users_stat[bucket_stat["owner"]] = {
bucket: bucket_stat["usage"]["rgw.main"]}
else:
users_stat[bucket_stat["owner"]] = {
bucket: {}}
except:
pass
return users_stat
except IndexError:
return users_stat
except TypeError:
return get_users_stat(s3_servers)
except exceptions.ServerDown:
return get_users_stat(s3_servers)
def get_rgw_stat(server):
try:
rgwAdmin = RGWAdmin(settings.S3_ACCESS, settings.S3_SECRET,
server, secure=False)
if rgwAdmin.get_users():
return 1
else:
return 0
except:
return 0
def get_rgw_stat(server):
try:
rgwAdmin = RGWAdmin(settings.S3_CRED['access_key'],
settings.S3_CRED['secret_key'],
server, secure=False)
if rgwAdmin.get_users():
return server, 1
else:
return server, 0
except:
return server, 0
def get_rgw_stat(server):
try:
rgwAdmin = RGWAdmin(settings.S3_ACCESS,
settings.S3_SECRET,
server,
secure=False)
if rgwAdmin.get_users():
return 1
else:
return 0
except:
return 0
def updateBucketInfo():
conn = pymysql.connect("localhost", "root", "PASSWORD", "rgw_info")
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS user_info(user_id varchar(100) not null primary key ,"
"display_name varchar(100) ,max_buckets int ,bucketlist longtext not null,max_size_kb bigint ,"
"max_objects int ,user_quota_enable varchar(100) ,user_bucket_quota_enable varchar (100)) ")
cursor.execute("CREATE TABLE IF NOT EXISTS bucket_info(bucket_id varchar(100) not null primary key ,"
" usaged varchar(100) ,usaged_gb int,"
"num_objects int ,bucket_max_size_gb int ,max_objects int ,bucket_quota_enable varchar(100) )")
rgw = RGWAdmin(
access_key='*******************',
secret_key='*************************',
server='**.**.**.**:7480',
verify=False, secure=False)
userlist = rgw.get_users()
for i in range(len(userlist)):
userinfo = rgw.get_user(uid=userlist[i])
bucketlist = rgw.get_bucket(uid=userlist[i])
userquota = rgw.get_user_quota(uid=userlist[i])
userbucketquota = rgw.get_user_bucket_quota(uid=userlist[i])
bucketlist_str = ",".join(bucketlist)
print(bucketlist_str)
user_sql = "replace into user_info values ('%s','%s','%s','%s','%s','%s','%s','%s') " \
% (userinfo['user_id'],
userinfo['display_name'],
userinfo['max_buckets'],
bucketlist_str,
userquota['max_size_kb'],
userquota['max_objects'],
userquota['enabled'],
userbucketquota['enabled'])
cursor.execute(user_sql)
conn.commit()
print("userinfo:", userinfo)
print("bucketlist:", bucketlist)
for j in range(len(bucketlist)):
bucketinfo = rgw.get_bucket(bucket=bucketlist[j])
bucketquota = bucketinfo['bucket_quota']
try:
bucket_rgw = bucketinfo['usage']['rgw.main']
print(bucket_rgw)
except:
bucket_rgw = {'size_kb_actual': 0, 'num_objects': 0}
bucket_sql = "replace into bucket_info values ('%s','%s','%s','%s','%s','%s','%s') " \
% (bucketinfo['bucket'],
bucket_rgw['size_kb_actual'] / bucketquota['max_size_kb'],
bucket_rgw['size_kb_actual'] / 1024 / 1024,
bucket_rgw['num_objects'],
bucketquota['max_size_kb'] / 1024 / 1024,
bucketquota['max_objects'],
bucketquota['enabled'])
cursor.execute(bucket_sql)
conn.commit()
conn.close()
def __init__(self):
host = os.getenv("host")
access_key = os.getenv("access_key")
secret_key = os.getenv("secret_key")
self.__rgwAdmin = RGWAdmin(server=host,
access_key=access_key,
secret_key=secret_key,
verify=False)
def get_ceph_admin_client() -> RGWAdmin:
"""Return a configured RGWAdmin client"""
server = urlparse(settings.CEPH_HOST).netloc
rgw = RGWAdmin(
access_key=settings.CEPH_ACCESS_KEY,
secret_key=settings.CEPH_SECRET_KEY,
server=server,
)
return rgw
def getBucketInfo(uid, secrectkey):
rgw = RGWAdmin(
access_key='*******************************',
secret_key='*******************************',
server='**.**.**.**:7480',
verify=False, secure=False)
sub_secrect = rgw.get_user(uid=uid)['keys']['secret_key']
if secrectkey == sub_secrect:
conn = pymysql.connect("localhost", "root", "PASSWORD", "rgw_info")
cursor = conn.cursor()
search_sql = "select * from user_info where bucket_id = '%(bucket_id)s'"
cursor.execute(search_sql)
search_result = cursor.fetchmany()
print(search_result)
conn.close()
jsonData = []
for row in search_result:
result = {'bucket_id': row[0], 'usaged': row[1], 'usaged_gb': row[2], 'num_objects': row[3],
'bucket_max_size_gb': row[4], 'max_objects ': row[5], 'bucket_quota_enable': row[6]}
jsonData.append(result)
return result
def rgw_client(region_id: int):
"""
初始化rgw客户端
使用区域中的管理员key生成rgw客户端
"""
try:
b = BucketRegion.objects.get(reg_id=region_id)
except BucketRegion.DoesNotExist:
return
return RGWAdmin(
access_key=b.access_key,
secret_key=b.secret_key,
server=b.server[7:]
if b.server.startswith('http://') else b.server[8:],
secure=False if b.server.startswith('http://') else True,
verify=False,
timeout=5,
)
import schedule
import time
logger = logging.getLogger('sync_log')
hdlr = logging.FileHandler('sync.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)
user = '******'
#Define the RGW Super User Connections
rgw_src = RGWAdmin(access_key='XXXXXXX',
secret_key='XXXXXXXXX',
server='XXXXX',
secure=False) #Add keys here of rgwadmin admin account
rgw_dst = RGWAdmin(access_key='XXXXXXX',
secret_key='XXXXXXXXXX',
server='XXXXXXXx',
secure=False) #Add keys here of rgwadmin admin account
def reconcile():
bucket_src_dict = {}
bucket_dst_dict = {}
dBuckets = rgw_src.get_bucket(uid=user, stats=True)
for bucket in dBuckets:
#print bucket["usage"]["rgw.main"]["num_objects"]
help="endpoint url for s3 object storage",
required=True)
parser.add_argument('-a',
'--access-key',
help='access key for s3 object storage',
required=True)
parser.add_argument('-s',
'--secret-key',
help='secret key for s3 object storage',
required=True)
args = parser.parse_args()
rgw = RGWAdmin(access_key=args.access_key,
secret_key=args.secret_key,
server=args.endpoint_url,
secure=False,
verify=False)
@app.route("/createuser", methods=['GET', 'POST'])
def createuser():
if request.method == 'POST':
result = request.form
username = request.args.get('uid')
displayname = request.args.get('displayname')
u = rgw.create_user(uid=username, display_name=displayname)
realm_dict[username] = 'realm1'
return u
from rgwadmin import RGWAdmin
ACCESS_KEY = 'AAT30ADPPNWA0FSLMAPJ'
SECRET_KEY = '7D3fgmOoCzix6qwe3KM2DmbSmaWGDZbCH4jBHqiO'
rgw = RGWAdmin(access_key=ACCESS_KEY,
secret_key=SECRET_KEY,
server='172.16.70.31:7480',
secure=False)
rgw.get_users()
def get_rgw_conn(self, access_key, secret_key, ip_addr):
return RGWAdmin(access_key=access_key,
secret_key=secret_key,
server=ip_addr,
secure=False)
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from django.http import JsonResponse
from django.conf import settings
from django.shortcuts import render
from django.shortcuts import redirect
from rgwadmin import RGWAdmin
s3_servers = list(settings.S3_SERVERS)
rgwAdmin = RGWAdmin(settings.S3_ACCESS,
settings.S3_SECRET,
s3_servers.pop(0),
secure=False)
def param(request, key):
if key not in request.GET:
raise Exception(key + "not found in request")
return request.GET[key]
def get_user_info(username, public=False):
userinfo = rgwAdmin.get_user(username)
userinfo.update({
"user_quota": rgwAdmin.get_quota(username, "user"),
"bucket_quota": rgwAdmin.get_quota(username, "bucket")
})
def test_exec(config):
io_info_initialize = IOInfoInitialize()
basic_io_structure = BasicIOInfoStructure()
io_info_initialize.initialize(basic_io_structure.initial())
umgmt = UserMgmt()
host, ip = utils.get_hostname_ip()
port = utils.get_radosgw_port_no()
hostname = str(ip) + ":" + str(port)
log.info(hostname)
# preparing data
admin_api_user = "******" + randomString()
log.info(admin_api_user)
user_info = umgmt.create_rest_admin_user(user_id=admin_api_user,
displayname=admin_api_user)
rgw = RGWAdmin(access_key=user_info['access_key'],
secret_key=user_info['secret_key'],
server=hostname,
secure=False,
verify=False)
api_user = "******" + randomString()
log.info(api_user)
for uc in range(config.user_count):
#Create User
data = rgw.create_user(uid=api_user,
display_name=api_user,
email=api_user + '@abc.xyz')
log.info("User created successfully")
log.info(data)
log.info('verification starts')
op = utils.exec_shell_cmd("radosgw-admin user info --uid %s" %
api_user)
json_doc = json.loads(op)
log.info(json_doc)
v = verify_user(data, json_doc)
if v is False:
test_info.failed_status('test failed')
sys.exit(1)
log.info("Verification for create operation completed")
#Update User
data = rgw.modify_user(uid=api_user,
display_name=api_user + "_11",
email=api_user + '*****@*****.**')
log.info("User Updated successfully")
log.info(data)
log.info('verification starts')
op = utils.exec_shell_cmd("radosgw-admin user info --uid %s" %
api_user)
json_doc = json.loads(op)
log.info(json_doc)
v = verify_user(data, json_doc)
if v is False:
test_info.failed_status('test failed')
sys.exit(1)
log.info("Verification for Update operation completed")
#delete User
data = rgw.remove_user(uid=api_user, purge_data=False)
log.info(data)
log.info("User removed")
op = utils.exec_shell_cmd("radosgw-admin user list")
json_doc = json.loads(op)
if api_user in json_doc:
test_info.failed_status('test failed')
sys.exit(1)
log.info("Verification for Delete operation completed")
# check for any crashes during the execution
crash_info = reusable.check_for_crash()
if crash_info:
raise TestExecError("ceph daemon crash found!")
import boto
import boto.s3.connection
from rgwadmin import RGWAdmin
# Users and Buckets from the primary server are created in the secondary server
rgw_sa = RGWAdmin(access_key=<access_key>, secret_key=<secret_key>,
server=<primary_server_endpoint>, secure=False) #Add keys here of rgwadmin admin account and endpoint
rgw_samit = RGWAdmin(access_key=<access_key>, secret_key=<secret_key>,
server=<secondary_server_endpoint>, secure=False) #Add keys here of rgwadmin admin account and endpoint
sa_users=rgw_sa.get_users()
samit_users=rgw_samit.get_users()
users={} #dict of the users with their access and secret keys
for user in sa_users:
user_details=rgw_sa.get_user(user)
users[user]={}
users[user]['ak']=user_details['keys'][0]['access_key']
users[user]['sk']=user_details['keys'][0]['secret_key']
if user not in samit_users:
rgw_samit.create_user(uid=user, display_name=user_details['display_name'],
email=user_details['email'],access_key=user_details['keys'][0]['access_key'],
secret_key=user_details['keys'][0]['secret_key'], generate_key=False)
print "User created in SAmit", user
count=1
for user in users.keys():
access_key=users[user]['ak']
secret_key=users[user]['sk']
print count,user
