def sync_domains(operator=None):
'''
同步域名
'''
aliyun_key_ctl.get_enabled_aliyun_key()
params = {}
data = {
'name': '同步域名',
'typ': 'sync_domain',
'params': params,
}
berry_ctl.create_berry(**data)
def sync_rockets(operator=None):
'''
同步Rocket
'''
aliyun_key_ctl.get_enabled_aliyun_key()
params = {}
data = {
'name': '同步Rocket',
'typ': 'sync_rocket',
'input_params': params,
'operator': operator,
}
berry_ctl.create_berry(**data)
def sync_slbs(operator=None):
'''
同步SLB
'''
# 先进行是否存在阿里云Key判断
aliyun_key_ctl.get_enabled_aliyun_key()
params = {}
data = {
'name': '同步SLB',
'typ': 'sync_slb',
'params': params,
}
berry_ctl.create_berry(**data)
def sync_zones():
'''
同步可用区
'''
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunManager(key, secret, 'cn-beijing')
region_objs = RegionModel.objects.all()
data_list = []
for region_obj in region_objs:
zones = ali_cli.get_zones(region_obj.instance_id)['data_list']
for zone in zones:
instance_id = zone.get('ZoneId')
data = {
'region_id': region_obj.id,
'name': zone.get('LocalName'),
'instance_id': instance_id,
}
obj = ZoneModel.objects.filter(region_id=region_obj.id, instance_id=instance_id).first()
# 存在就更新，不存在就创建
if obj:
base_ctl.update_obj(ZoneModel, obj.id, data)
else:
data_list.append(data)
data_list.append(data)
base_ctl.create_objs(ZoneModel, data_list)
def sync_domain_records():
'''
同步域名解析记录
'''
with transaction.atomic():
domain_objs = DomainModel.objects.all()
old_ids = DomainRecordModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
record_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunDNS(key, secret, 'cn-beijing')
for domain_obj in domain_objs:
page_num = 1
page_size = 50
while True:
query = {
'domain_name': domain_obj.name,
'page_num': page_num,
'page_size': page_size,
}
data = ali_cli.get_domain_records(**query)
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
instance_id = data.get('RecordId')
value = data.get('Value')
rr = data.get('RR')
typ = data.get('Type')
enabled = True if data.get('Status') == 'ENABLE' else False
query = {
'domain_id': domain_obj.id,
'instance_id': instance_id,
}
obj = DomainRecordModel.objects.filter(**query).first()
data = query
data['value'] = value
data['rr'] = rr
data['typ'] = typ
# 这里name和fullname就是冗余存储，为了方便查询
data['name'] = domain_obj.name
data['fullname'] = rr + '.' + domain_obj.name
data['enabled'] = enabled
if not obj:
record_list.append(data)
else:
base_ctl.update_obj(DomainRecordModel, obj.id, data)
existed_ids.append(obj.id)
if total <= page_num * page_size:
break
page_num += 1
base_ctl.create_objs(DomainRecordModel, record_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(DomainRecordModel, deleted_ids)
sync_domain_record_asset()
def sync_slb_backend_servers():
'''
同步SLB默认服务器组服务器
'''
with transaction.atomic():
slb_objs = SlbModel.objects.all()
query = {
'server_group__typ': SlbServerGroupModel.TYP_DEFAULT,
}
old_ids = SlbServerGroupEcsModel.objects.filter(**query).values_list(
'id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
ecs_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunSLB(key, secret, 'cn-beijing')
for slb_obj in slb_objs:
group_obj = SlbServerGroupModel.objects.filter(slb_id=slb_obj.id)\
.filter(typ=SlbServerGroupModel.TYP_DEFAULT).first()
if not group_obj:
data = {
'slb_id': slb_obj.id,
'name': 'default',
'instance_id': 'default',
'typ': SlbServerGroupModel.TYP_DEFAULT,
}
group_obj = base_ctl.create_obj(SlbServerGroupModel, data)
ali_cli.reset_region(slb_obj.region_id)
ecses = ali_cli.get_slb_info(
slb_obj.instance_id).get('backend_servers')
for ecs in ecses:
ecs_instance_id = ecs.get('ServerId')
ecs_obj = EcsModel.objects.filter(
instance_id=ecs_instance_id).first()
weight = ecs.get('Weight')
obj = SlbServerGroupEcsModel.objects.filter(slb_id=slb_obj.id)\
.filter(server_group_id=group_obj.id, ecs_id=ecs_obj.id).first()
data = {
'slb_id': slb_obj.id,
'server_group_id': group_obj.id,
'ecs_id': ecs_obj.id,
'weight': weight,
}
if not obj:
ecs_list.append(data)
else:
base_ctl.update_obj(SlbServerGroupEcsModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(SlbServerGroupEcsModel, ecs_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
if deleted_ids:
base_ctl.delete_objs(SlbServerGroupEcsModel, deleted_ids)
sync_slb_vserver_groups()
def sync_rds_databases():
'''
同步RDS Database
'''
with transaction.atomic():
rds_objs = RdsModel.objects.all()
old_ids = RdsDatabaseModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
database_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunRDS(key, secret, 'cn-beijing')
for rds_obj in rds_objs:
ali_cli.reset_region(rds_obj.region_id)
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
'instance_id': rds_obj.instance_id,
}
data = ali_cli.get_rds_databases(**query)
data_list = data.get('data_list')
for data in data_list:
instance_id = data.get('DBName')
name = data.get('DBName')
desc = data.get('DBDescription')
# 这里先存一下关联account信息，之后省得再用接口获取
accounts = data.get('Accounts').get('AccountPrivilegeInfo')
query = {
'rds_id': rds_obj.id,
'instance_id': instance_id,
}
obj = RdsDatabaseModel.objects.filter(**query).first()
data = query
data['name'] = name
data['desc'] = desc
data['accounts'] = json.dumps(accounts)
if not obj:
database_list.append(data)
else:
base_ctl.update_obj(RdsDatabaseModel, obj.id, data)
existed_ids.append(obj.id)
if page_size > len(data_list):
break
page_num += 1
base_ctl.create_objs(RdsDatabaseModel, database_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RdsDatabaseModel, deleted_ids)
sync_rds_databases_accounts()
def sync_rdses():
'''
同步RDS
'''
with transaction.atomic():
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
regions = region_ctl.get_regions(
status=RegionModel.ST_ENABLE)['data_list']
# 记录原来已经创建过的RDS，用于之后删除已经不存在的使用
old_ids = RdsModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
# 用来存储仍然可以查到的RDS
existed_ids = []
# 记录需要新创建的RDS信息，用于批量创建
rds_list = []
# 每次使用都先使用默认的地域初始化，其实可以在类里增加默认值，但是没有增加默认值是为了更明确知道在干什么
ali_cli = AliyunRDS(key, secret, 'cn-beijing')
for region in regions:
region_id = region.get('instance_id')
ali_cli.reset_region(region_id)
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
}
data = ali_cli.get_rdses(**query)
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
data = format_rds_data(data)
instance_id = data.get('instance_id')
attribute = ali_cli.get_rds_attribute(instance_id)
if attribute:
data['connection'] = attribute.get('ConnectionString')
obj = RdsModel.objects.filter(
instance_id=instance_id).first()
if obj:
base_ctl.update_obj(RdsModel, obj.id, data)
existed_ids.append(obj.id)
else:
rds_list.append(data)
if total <= page_num * page_size:
break
page_num += 1
base_ctl.create_objs(RdsModel, rds_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RdsModel, deleted_ids)
sync_rds_accounts()
def sync_slb_vserver_group_backend_servers():
'''
同步虚拟服务器组后端服务器
'''
with transaction.atomic():
query = {
'typ': SlbServerGroupModel.TYP_VSERVER,
}
group_objs = SlbServerGroupModel.objects.filter(**query).all()
query = {
'server_group__typ': SlbServerGroupModel.TYP_VSERVER,
}
old_ids = SlbServerGroupEcsModel.objects.filter(**query).values_list(
'id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
ecs_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunSLB(key, secret, 'cn-beijing')
for group_obj in group_objs:
ali_cli.reset_region(group_obj.slb.region_id)
ecses = ali_cli.get_vserver_group_backend_servers(
group_obj.instance_id).get('data_list')
for ecs in ecses:
ecs_instance_id = ecs.get('ServerId')
weight = ecs.get('Weight')
ecs_obj = EcsModel.objects.filter(
instance_id=ecs_instance_id).first()
query = {
'slb_id': group_obj.slb_id,
'server_group_id': group_obj.id,
'ecs_id': ecs_obj.id,
}
obj = SlbServerGroupEcsModel.objects.filter(**query).first()
data = {
'slb_id': group_obj.slb_id,
'server_group_id': group_obj.id,
'ecs_id': ecs_obj.id,
'weight': weight,
}
if not obj:
ecs_list.append(data)
else:
base_ctl.update_obj(SlbServerGroupEcsModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(SlbServerGroupEcsModel, ecs_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
if deleted_ids:
base_ctl.delete_objs(SlbServerGroupEcsModel, deleted_ids)
def sync_slbs():
'''
同步SLB
'''
with transaction.atomic():
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
regions = region_ctl.get_regions(
status=RegionModel.ST_ENABLE)['data_list']
# 记录原来已经创建过的SLB，用于之后删除已经不存在的使用
old_ids = SlbModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
# 用来存储仍然可以查到的SLB
existed_ids = []
# 记录需要新创建的SLB信息，用于批量创建
slb_list = []
# 每次使用都先使用默认的地域初始化，其实可以在类里增加默认值，但是没有增加默认值是为了更明确知道在干什么
ali_cli = AliyunSLB(key, secret, 'cn-beijing')
for region in regions:
region_id = region.get('instance_id')
ali_cli.reset_region(region_id)
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
}
data = ali_cli.get_slbs(**query)
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
data = format_slb_data(data)
instance_id = data.get('instance_id')
obj = SlbModel.objects.filter(
instance_id=instance_id).first()
if obj:
base_ctl.update_obj(SlbModel, obj.id, data)
existed_ids.append(obj.id)
else:
slb_list.append(data)
if total <= page_num * page_size:
break
page_num += 1
base_ctl.create_objs(SlbModel, slb_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
if deleted_ids:
base_ctl.delete_objs(SlbModel, deleted_ids)
sync_slb_backend_servers()
def sync_rds_accounts():
'''
同步RDS账号
'''
with transaction.atomic():
rds_objs = RdsModel.objects.all()
old_ids = RdsAccountModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
account_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunRDS(key, secret, 'cn-beijing')
for rds_obj in rds_objs:
ali_cli.reset_region(rds_obj.region_id)
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
'instance_id': rds_obj.instance_id,
}
data = ali_cli.get_rds_accounts(**query)
data_list = data.get('data_list')
for data in data_list:
username = data.get('AccountName')
query = {
'rds_id': rds_obj.id,
'username': username,
}
obj = RdsAccountModel.objects.filter(**query).first()
data = query
if not obj:
account_list.append(data)
else:
base_ctl.update_obj(RdsAccountModel, obj.id, data)
existed_ids.append(obj.id)
if page_size > len(data_list):
break
page_num += 1
base_ctl.create_objs(RdsAccountModel, account_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RdsAccountModel, deleted_ids)
sync_rds_databases()
def sync_slb_vserver_groups():
'''
同步SLB虚拟服务器组
'''
with transaction.atomic():
slb_objs = SlbModel.objects.all()
old_ids = SlbServerGroupModel.objects.filter(typ=SlbServerGroupModel.TYP_VSERVER)\
.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
group_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunSLB(key, secret, 'cn-beijing')
for slb_obj in slb_objs:
ali_cli.reset_region(slb_obj.region_id)
groups = ali_cli.get_vserver_groups(
slb_obj.instance_id).get('data_list')
for group in groups:
group_instance_id = group.get('VServerGroupId')
group_name = group.get('VServerGroupName')
query = {
'slb_id': slb_obj.id,
'typ': SlbServerGroupModel.TYP_VSERVER,
'instance_id': group_instance_id,
}
group_obj = SlbServerGroupModel.objects.filter(**query).first()
data = {
'slb_id': slb_obj.id,
'instance_id': group_instance_id,
'name': group_name,
'typ': SlbServerGroupModel.TYP_VSERVER,
}
if not group_obj:
group_list.append(data)
else:
group_obj = base_ctl.update_obj(SlbServerGroupModel,
group_obj.id, data)
existed_ids.append(group_obj.id)
base_ctl.create_objs(SlbServerGroupModel, group_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
if deleted_ids:
base_ctl.delete_objs(SlbServerGroupModel, deleted_ids)
sync_slb_vserver_group_backend_servers()
def sync_redis_accounts():
'''
同步Redis账号
'''
with transaction.atomic():
redis_objs = RedisModel.objects.all()
old_ids = RedisAccountModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
account_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunRedis(key, secret, 'cn-beijing')
for redis_obj in redis_objs:
ali_cli.reset_region(redis_obj.region_id)
query = {
'instance_id': redis_obj.instance_id,
}
data = ali_cli.get_redis_accounts(**query)
data_list = data.get('data_list')
for data in data_list:
username = data.get('AccountName')
typ = data.get('AccountType')
status = data.get('AccountStatus')
privilege = data.get('DatabasePrivileges').get(
'DatabasePrivilege')[0].get('AccountPrivilege')
query = {
'redis_id': redis_obj.id,
'username': username,
}
obj = RedisAccountModel.objects.filter(**query).first()
data = query
data['typ'] = typ
data['status'] = status
data['privilege'] = privilege
if not obj:
account_list.append(data)
else:
base_ctl.update_obj(RedisAccountModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(RedisAccountModel, account_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RedisAccountModel, deleted_ids)
def sync_domains():
'''
同步Domain
'''
with transaction.atomic():
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
# 记录原来已经创建过的Domain，用于之后删除已经不存在的使用
old_ids = DomainModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
# 用来存储仍然可以查到的Domain
existed_ids = []
# 记录需要新创建的Domain信息，用于批量创建
domain_list = []
# 每次使用都先使用默认的地域初始化，其实可以在类里增加默认值，但是没有增加默认值是为了更明确知道在干什么
ali_cli = AliyunDNS(key, secret, 'cn-beijing')
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
}
data = ali_cli.get_domains(**query)
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
data = format_domain_data(data)
instance_id = data.get('instance_id')
obj = DomainModel.objects.filter(instance_id=instance_id).first()
if obj:
base_ctl.update_obj(DomainModel, obj.id, data)
existed_ids.append(obj.id)
else:
domain_list.append(data)
if total <= page_num * page_size:
break
page_num += 1
base_ctl.create_objs(DomainModel, domain_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(DomainModel, deleted_ids)
sync_domain_records()
def sync_mongo_accounts():
'''
同步Mongo账号
'''
with transaction.atomic():
mongo_objs = MongoModel.objects.all()
old_ids = MongoAccountModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
account_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunMongo(key, secret, 'cn-beijing')
for mongo_obj in mongo_objs:
ali_cli.reset_region(mongo_obj.region_id)
query = {
'instance_id': mongo_obj.instance_id,
}
data = ali_cli.get_mongo_accounts(**query)
data_list = data.get('data_list')
for data in data_list:
username = data.get('AccountName')
typ = data.get('CharacterType')
status = data.get('AccountStatus')
remark = data.get('AccountDescription')
query = {
'mongo_id': mongo_obj.id,
'username': username,
}
obj = MongoAccountModel.objects.filter(**query).first()
data = query
data['typ'] = typ
data['status'] = status
data['remark'] = remark
if not obj:
account_list.append(data)
else:
base_ctl.update_obj(MongoAccountModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(MongoAccountModel, account_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(MongoAccountModel, deleted_ids)
def sync_rocket_groups():
'''
同步Rocket Group
'''
with transaction.atomic():
rocket_objs = RocketModel.objects.all()
old_ids = RocketGroupModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
group_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunONS(key, secret, 'cn-beijing')
for rocket_obj in rocket_objs:
ali_cli.reset_region(rocket_obj.region_id)
query = {
'instance_id': rocket_obj.instance_id,
}
data = ali_cli.get_rocket_groups(**query)
data_list = data.get('data_list')
for data in data_list:
name = data.get('GroupId')
typ = data.get('GroupType')
remark = data.get('Remark')
query = {
'rocket_id': rocket_obj.id,
'name': name,
}
obj = RocketGroupModel.objects.filter(**query).first()
data = query
data['typ'] = typ
data['remark'] = remark
if not obj:
group_list.append(data)
else:
base_ctl.update_obj(RocketGroupModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(RocketGroupModel, group_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RocketGroupModel, deleted_ids)
def sync_regions():
'''
同步地域
'''
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunManager(key, secret, 'cn-beijing')
regions = ali_cli.get_regions()['data_list']
data_list = []
for region in regions:
instance_id = region.get('RegionId')
data = {
'name': region.get('LocalName'),
'instance_id': instance_id,
'endpoint': region.get('RegionEndpoint'),
}
obj = RegionModel.objects.filter(instance_id=instance_id).first()
# 存在就更新，不存在就创建
if obj:
base_ctl.update_obj(RegionModel, obj.id, data)
else:
data_list.append(data)
base_ctl.create_objs(RegionModel, data_list)
def sync_rockets():
'''
同步Rocket
'''
with transaction.atomic():
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
regions = region_ctl.get_regions(
status=RegionModel.ST_ENABLE)['data_list']
# 记录原来已经创建过的Rocket，用于之后删除已经不存在的使用
old_ids = RocketModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
# 用来存储仍然可以查到的Rocket
existed_ids = []
# 记录需要新创建的Rocket信息，用于批量创建
rocket_list = []
# 每次使用都先使用默认的地域初始化，其实可以在类里增加默认值，但是没有增加默认值是为了更明确知道在干什么
ali_cli = AliyunONS(key, secret, 'cn-beijing')
for region in regions:
region_id = region.get('instance_id')
ali_cli.reset_region(region_id)
data = ali_cli.get_rockets()
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
data = format_rocket_data(data)
data['region_id'] = region_id
instance_id = data.get('instance_id')
obj = RocketModel.objects.filter(
instance_id=instance_id).first()
if obj:
base_ctl.update_obj(RocketModel, obj.id, data)
existed_ids.append(obj.id)
else:
rocket_list.append(data)
base_ctl.create_objs(RocketModel, rocket_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RocketModel, deleted_ids)
sync_rocket_topics()
