def slow_DBCheck(self):
for region, instance in self.get_DBInstance().items():
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
request.set_action_name("DescribeSlowLogs")
request.set_DBInstanceId(instance)
request.add_query_param("StartTime", self.startTime)
request.add_query_param("EndTime",
time.strftime("%Y-%m-%dZ", time.gmtime()))
clt = client.AcsClient(self.accesskey, self.accessSecret, region)
try:
result = clt.do_action(request)
data = json.loads(result)["Items"]['SQLSlowLog']
if data:
li = []
for i in range(len(data)):
MySQLTotalExecutionTimes = data[i][
'MySQLTotalExecutionTimes']
MaxExecutionTime = data[i]['MaxExecutionTime']
SQLText = data[i]['SQLText']
DBName = data[i]['DBName']
CreateTime = data[i]['CreateTime']
li.append({
'SQLText': SQLText,
'DBName': DBName,
'MySQLTotalExecutionTimes':
MySQLTotalExecutionTimes,
'MaxExecutionTime': MaxExecutionTime,
'CreateTime': CreateTime
})
return li
else:
continue
except Exception:
pass
def GetRdsList():
RdsRequest = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
RdsRequest.set_accept_format('json')
RdsInfo = clt.do_action_with_exception(RdsRequest)
for RdsInfoJson in (json.loads(RdsInfo))['Items']['DBInstance']:
DBInstanceIdList.append(RdsInfoJson['DBInstanceId'])
print DBInstanceIdList
def sync_rds_info(request):
temp_name = "monitor/monitor-header.html"
client = AcsClient(ACCESS_KEY_ID, ACCESS_KEY_SECRET, 'cn-shanghai')
rds_env = {
'prd': 'Production',
'pp': 'Pre-production',
'qa': 'Quality Assurance',
'dev': 'Develop',
'others': 'Others'
}
if RdsGroup.objects.all().count() != 0:
RdsGroup.objects.all().delete()
for name, desc in rds_env.items():
RdsGroup(name=name, desc=desc).save()
else:
for name, desc in rds_env.items():
RdsGroup(name=name, desc=desc).save()
req = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
req.set_PageSize(100)
resp = json.loads(client.do_action_with_exception(req), encoding='utf-8')
for db in resp.get('Items').get('DBInstance'):
db_instance_id = db.get('DBInstanceId')
db_instance_description = db.get('DBInstanceDescription')
db_instance_type = db.get('DBInstanceType')
db_instance_class = db.get('DBInstanceClass')
zone_id = db.get('ZoneId')
connection_mode = db.get('ConnectionMode')
engine = db.get('Engine')
engine_version = db.get('EngineVersion')
rds_group = db_instance_description.split('-')[-1]
if rds_group in rds_env:
group = RdsGroup.objects.get(name=rds_group)
Rds(db_instance_id=db_instance_id,
db_instance_description=db_instance_description,
db_instance_type=db_instance_type,
db_instance_class=db_instance_class,
zone_id=zone_id,
connection_mode=connection_mode,
engine=engine,
engine_version=engine_version,
rds_group=group).save()
else:
group = RdsGroup.objects.get(name='others')
Rds(db_instance_id=db_instance_id,
db_instance_description=db_instance_description,
db_instance_type=db_instance_type,
db_instance_class=db_instance_class,
zone_id=zone_id,
connection_mode=connection_mode,
engine=engine,
engine_version=engine_version,
rds_group=group).save()
return render(request, "monitor/rds_index.html", locals())
def get_DescribeDBInstances(self):
Total = self.get_TotalPages('DescribeInstances')
for page in xrange(1, Total + 1):
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
request.set_PageSize('10')
request.set_PageNumber(page)
result = json.loads(self.client.do_action_with_exception(request))
yield result['Items']['DBInstance'] if result['Items'] else []
def query_rds_list(RegionId):
# 设置参数
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
request.add_query_param('RegionId', RegionId)
# 发起请求
response = clt.do_action(request)
json_response = json.loads(response)
return json_response['Items']['DBInstance'][0]
def request_get_instance(self, instance_id):
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
self.request_to_json(request)
request.add_query_param('RegionId', 'cn-hangzhou')
request.add_query_param('DBInstanceId', instance_id)
try:
response = self.clt.do_action_with_exception(request)
except:
return {}
return self.get_json_results(response).get('Items').get('DBInstance')[0]
def request_get_page_number(self):
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.add_query_param('PageNumber', 1)
request.add_query_param('PageSize', 1)
try:
response = self.clt.do_action_with_exception(request)
except:
return {}
result = self.get_json_results(response)
return int(result.get('TotalRecordCount')/settings.ALIYUN_PAGESIZE)+1
def get_DBid(self):
try:
for zone in self.zone:
client = AcsClient('', '', zone)
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest(
)
request.set_accept_format('json')
response = json.loads(client.do_action_with_exception(request))
self.DBid.append(response['Items']['DBInstance'])
except Exception as e:
raise e
def getInstanceListByID(self, instanceIds):
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_PageSize(100)
request.set_InstanceIds(instanceIds)
request.set_accept_format('json')
try:
result = self.clt.do_action_with_exception(request)
result = json.loads(result)
return True, result['Instances']['Instance']
except Exception, ex:
return False, str(ex)
def request_get_instances(self, page):
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
self.request_to_json(request)
request.add_query_param('RegionId', 'cn-hangzhou')
request.add_query_param('PageNumber', page)
request.add_query_param('PageSize', settings.ALIYUN_PAGESIZE)
try:
response = self.clt.do_action_with_exception(request)
except:
return {}
return self.get_json_results(response).get('Items').get('DBInstance')
def get_rds_list(self):
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
result = self._request(request)
instance_list = []
for item in result['Items']['DBInstance']:
instance_list.append(item['DBInstanceId'])
#print json.dumps(result,sort_keys=True,indent=4, separators=(',', ': '))
return instance_list
def aliyun_rds(self):
clt = client.AcsClient(self.SecretId, self.SecretKey, self.RegionId)
# 设置参数
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
request.add_query_param('PageNumber', 1)
request.add_query_param('PageSize', 100)
# 发起请求
request.add_query_param('RegionId', self.RegionId)
response = clt.do_action(request)
str_response = json.loads(response)
return str_response
def GetRdsList():
RdsRequest = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
RdsRequest.set_accept_format('json')
RdsRequest.set_DBInstanceId('rm-bp18144qkyj20731o')
RdsInfo = json.loads(client.do_action_with_exception(RdsRequest))
print RdsInfo
for RdsInfoJson in RdsInfo["Items"]["DBInstance"]:
DBInstanceIdDict = {}
DBInstanceIdDict["{#DBINSTANCEID}"] = RdsInfoJson['DBInstanceId']
#print DBInstanceIdDict["{#DBINSTANCEID}"]
DBInstanceIdDict["{#InsId}"] = RdsInfoJson['InsId']
#print DBInstanceIdDict["{#InsId}"]
DBInstanceIdList.append(DBInstanceIdDict)
def get_db_instance(self, name=None):
"""
返回实例具名元祖组成的列表
:param name: 指定别名，则只返回该实例
:return: [db_instance(name='report_readonly', id='rr-bp1se0dwk3vueq89k')]
"""
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
response = self.clt.do_action_with_exception(request)
instance = namedtuple('db_instance', 'name id')
return [
instance(x['DBInstanceDescription'], x['DBInstanceId'])
for x in json.loads(response)['Items']['DBInstance']
if x['DBInstanceDescription'] == name or not name
]
def rds_list(): #定义rds_list函数，用于获取所有的rds实例ID
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_Engine('MySQL') #请求的RDS类型
request.set_PageSize(100) #允许请求的RDS最大数量
result = json.loads(clt.do_action_with_exception(request)).get(
'Items').get('DBInstance')
for line in result:
names[line.get('DBInstanceId')] = line.get('DBInstanceDescription')
alias = line.get('DBInstanceDescription')
if line.get('ExpireTime') != '':
expiretime = line.get('ExpireTime')
days = DayS(expiretime)
print "还有%s" % days.days_values() + "天到期", "实例:", line.get(
'DBInstanceDescription')
async def get_instances(self, region):
"""
Get all instances
:return: a list of all instances
"""
client = get_client(credentials=self._credentials, region=region)
response = await get_response(
client=client,
request=DescribeDBInstancesRequest.DescribeDBInstancesRequest())
if response:
return response['Items']['DBInstance']
else:
return []
def GetRdsList():
RdsRequest = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
RdsRequest.set_accept_format('json')
# RdsInfo = clt.do_action(RdsRequest)
RdsInfo = clt.do_action_with_exception(RdsRequest)
for RdsInfoJson in (json.loads(RdsInfo))['Items']['DBInstance']:
DBInstanceIdDict = {}
try:
DBInstanceIdDict["{#DBINSTANCEID}"] = RdsInfoJson['DBInstanceId']
DBInstanceIdDict["{#DBINSTANCEDESCRIPTION}"] = RdsInfoJson['DBInstanceDescription']
DBInstanceIdList.append(DBInstanceIdDict)
except Exception, e:
print Exception, ":", e
print "Please check the RDS alias !Alias must not be the same as DBInstanceId！！！"
def fetch_RDSs(page):
from aliyunsdkrds.request.v20140815 import DescribeDBInstancesRequest
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
request.add_query_param('RegionId', 'cn-hangzhou')
request.add_query_param('PageNumber', page)
request.add_query_param('PageSize', conf.ALIYUN_PAGESIZE)
try:
response = clt.do_action_with_exception(request)
except:
return {}
data = json.loads(response.decode('utf-8'))
return data['Items']['DBInstance']
def fetch_RDSPage():
from aliyunsdkrds.request.v20140815 import DescribeDBInstancesRequest
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
request.add_query_param('RegionId', 'cn-hangzhou')
request.add_query_param('PageSize', 1)
request.add_query_param('PageNumber', 1)
try:
response = clt.do_action_with_exception(request)
except:
return {}
data = json.loads(response.decode('utf-8'))
return data['TotalRecordCount']
def getInstanceList(self, pageNumber, pageSize):
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_PageNumber(pageNumber)
request.set_PageSize(pageSize)
request.set_accept_format('json')
try:
result = self.clt.do_action_with_exception(request)
print result
result = json.loads(result)
data = {
'count': result['TotalRecordCount'],
'results': result['Items']['DBInstance']
}
return True, data
except Exception, ex:
return False, str(ex)
def fetch_MongoDBs(page):
from aliyunsdkdds.request.v20151201 import DescribeDBInstancesRequest
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
request.add_query_param('RegionId', 'cn-hangzhou')
request.add_query_param('PageNumber', page)
request.add_query_param('PageSize', 30)
try:
response = clt.do_action_with_exception(request)
except:
return {}
data = json.loads(response.decode('utf-8'))
return data['DBInstances']['DBInstance']
def get_rds(self, pageStart=0, pageSize=0):
"""
获取rds列表
:param pageStart:
:param pageSize:
:return:
"""
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
if pageStart:
request.set_PageNumber(pageStart)
if pageSize:
request.set_PageSize(pageSize) # 设置每页返回多少，默认为10条
request.set_accept_format("json")
result = json.loads(self.clt.do_action(request))
print(result)
return result
def get_DBInstance(self):
DBInstance_dic = {}
for regionId in self.get_DescribeRegions():
clt = client.AcsClient(self.accesskey, self.accessSecret, regionId)
request1 = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request1.set_action_name("DescribeDBInstances")
request1.set_accept_format('json')
try:
result = clt.do_action(request1)
data = json.loads(result)
items = data['Items']['DBInstance']
if items:
DBInstance_dic[regionId] = items[0]['DBInstanceId']
else:
continue
except Exception:
pass
return DBInstance_dic
def update_Rds():
Rds_Info = {}
client = AcsClient(Config.configs['AliAK']['AccessKey'],
Config.configs['AliAK']['AccessKeySecret'], RegionId)
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_PageSize(100)
response = client.do_action_with_exception(request)
RdsInfo = json.loads(response)
for DBInstance in RdsInfo['Items']['DBInstance']:
request = DescribeDBInstanceAttributeRequest.DescribeDBInstanceAttributeRequest(
)
request.set_DBInstanceId(DBInstance['DBInstanceId'])
response = client.do_action_with_exception(request)
Rds_Info[DBInstance['DBInstanceId']] = json.loads(
response.decode('utf-8'))['Items']['DBInstanceAttribute'][0]
conn.set('Rds_Info', json.dumps(Rds_Info))
def query_mysql(self):
try:
for zone in current_app.config['ALIYUN_ZONE']:
client = AcsClient(current_app.config['ALIYUN_ACCESS_KEYID'], current_app.config['ALIYUN_ACCESS_KEY_SECRET'], zone)
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
response = json.loads(client.do_action_with_exception(request))
self.DBid.append(response['Items']['DBInstance'])
except Exception as e:
raise e
for zone in current_app.config['ALIYUN_ZONE']:
client = AcsClient(current_app.config['ALIYUN_ACCESS_KEYID'], current_app.config['ALIYUN_ACCESS_KEY_SECRET'],zone)
for ids in sum(self.DBid, []):
request = DescribeDBInstanceAttributeRequest.DescribeDBInstanceAttributeRequest()
request.set_accept_format('json')
request.set_DBInstanceId(ids['DBInstanceId'])
response = json.loads(client.do_action_with_exception(request))
self.DBinfo.append(response['Items']['DBInstanceAttribute'])
return self.DBinfo
def get_TotalPages(self, action_name):
Total = 0
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
request.set_PageSize('10')
result = json.loads(self.client.do_action_with_exception(request))
TotalCount = float(result['TotalRecordCount'])
PageSize = float(result['PageRecordCount'])
if TotalCount > PageSize:
try:
# TotalCount = float(result['TotalCount'])
# PageSize = float(result['PageSize'])
return int(round(TotalCount / PageSize))
except Exception as e:
print e.message
return Total
else:
return 1
def request_get_instances(self, page):
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
self.request_to_json(request)
request.add_query_param('RegionId', 'cn-hangzhou')
request.add_query_param('PageNumber', page)
request.add_query_param('PageSize', settings.ALIYUN_PAGESIZE)
try:
response = self.clt.do_action_with_exception(request)
except:
return {}
return self.get_json_results(response).get('Items').get('DBInstance')
# @staticmethod
# def get_aliyun_models(json_results):
# return {
# 'aliyun_id': json_results.get('DBInstanceId'),
# 'name': json_results.get('DBInstanceDescription')
# }
def get_rds_whitelistip_info(
self,
ip,
) -> typing.Generator:
"""
Get WhitelistIP info of RDS Data Security.
:param ip: IP to get info.
:return: Dict, {"DBInstanceId": ..., }.
"""
client = self.get_cloud_client()
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
response = client.do_action_with_exception(request)
for instance in json.loads(
(response.decode("utf-8")))["Items"]["DBInstance"]:
request_sub = DescribeDBInstanceIPArrayListRequest.DescribeDBInstanceIPArrayListRequest(
)
request_sub.set_DBInstanceId(instance["DBInstanceId"])
response_sub = client.do_action_with_exception(request_sub)
data_sub = json.loads((response_sub.decode("utf-8")))
if re.search(self.get_ip_pattern(ip), str(data_sub)):
for config in data_sub["Items"]["DBInstanceIPArray"]:
if re.search(self.get_ip_pattern(ip), str(config)):
whitelistip_info = {
"DBInstanceId":
instance["DBInstanceId"],
"DBInstanceIPArrayName":
config["DBInstanceIPArrayName"],
"DBInstanceIPArrayAttribute":
config["DBInstanceIPArrayAttribute"],
"WhitelistNetworkType":
config["WhitelistNetworkType"],
"SecurityIPType":
config["SecurityIPType"],
"CheckInfo":
config
}
logger.info(
"get_rds_whitelistip_info:\n\t{whitelistip_info}".
format(whitelistip_info=whitelistip_info))
yield whitelistip_info
break
def get_rds_instances(self, pageSize=100):
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
request.set_accept_format('json')
attributeRequest = DescribeDBInstanceAttributeRequest.DescribeDBInstanceAttributeRequest()
attributeRequest.set_accept_format('json')
for clt in self.clt_conn_list:
pageNumber = 1
while True:
request.set_query_params(dict(PageNumber=pageNumber, PageSize=pageSize))
clt_result = json.loads(clt.do_action_with_exception(request), encoding='utf-8')
region_result = clt_result['Items']['DBInstance']
pageNumber += 1
if len(region_result) == 0:
print(pageNumber, clt_result['PageRecordCount'])
break
for Instance in region_result:
attributeRequest.add_query_param("action_name", "DescribeDBInstanceAttribute")
attributeRequest.add_query_param("DBInstanceId", Instance['DBInstanceId'])
r = json.loads(clt.do_action_with_exception(attributeRequest))
attr = r['Items']['DBInstanceAttribute'][0]
expired_time = attr.get('ExpireTime')
if not expired_time:
expired_time = None
yield {
'instance_id': attr.get('DBInstanceId'),
'region': attr.get('RegionId'),
'instance_name': attr.get('DBInstanceDescription'),
'network_type': attr.get('InstanceNetworkType'),
'engine_version': attr.get('EngineVersion'),
'status': attr.get('DBInstanceStatus'),
'expired_time': expired_time,
'connection_string': attr.get('ConnectionString'),
'connections': attr.get('MaxConnections'),
'cpu': attr.get('DBInstanceCPU'),
'memory': attr.get('DBInstanceMemory'),
'iops': attr.get('MaxIOPS'),
'storage': attr.get('DBInstanceStorage')
}
def list_rds(self, name_only=False, **kwargs):
"""
获取实例列表
https://help.aliyun.com/document_detail/26232.html?spm=a2c4g.11186623.6.1322.16ff69b2We0YoY
:param name_only: 是否只返回name列表
:param kwargs:
:return:
"""
result_list = []
request = DescribeDBInstancesRequest.DescribeDBInstancesRequest()
regions = self.get_regions(id_only=True)
for region in regions:
request.set_PageSize(100)
request.set_PageNumber(1)
for k, v in kwargs.items():
request.add_query_param(k, v)
request.add_query_param('RegionId', region)
result = self.client.do_action(request)
result_list.extend(result.get('Items', {}).get('DBInstance', []))
total_count = float(result.get('TotalCount', 0))
if not total_count:
continue
page_count = float(result.get('PageSize', 1))
_p = total_count / page_count
if _p > 1:
for _p_num in range(2, int(_p) + 2):
# 防止限流
time.sleep(1)
request.set_PageNumber(_p_num)
result = self.client.do_action(request)
result_list.extend(
result.get('Items', {}).get('DBInstance', []))
if name_only:
result_list = [i['InstanceName'] for i in result_list]
return result_list
