def sync_domains(operator=None): ''' 同步域名 ''' aliyun_key_ctl.get_enabled_aliyun_key() params = {} data = { 'name': '同步域名', 'typ': 'sync_domain', 'params': params, } berry_ctl.create_berry(**data)
def sync_rockets(operator=None): ''' 同步Rocket ''' aliyun_key_ctl.get_enabled_aliyun_key() params = {} data = { 'name': '同步Rocket', 'typ': 'sync_rocket', 'input_params': params, 'operator': operator, } berry_ctl.create_berry(**data)
def sync_slbs(operator=None): ''' 同步SLB ''' # 先进行是否存在阿里云Key判断 aliyun_key_ctl.get_enabled_aliyun_key() params = {} data = { 'name': '同步SLB', 'typ': 'sync_slb', 'params': params, } berry_ctl.create_berry(**data)
def sync_zones(): ''' 同步可用区 ''' key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunManager(key, secret, 'cn-beijing') region_objs = RegionModel.objects.all() data_list = [] for region_obj in region_objs: zones = ali_cli.get_zones(region_obj.instance_id)['data_list'] for zone in zones: instance_id = zone.get('ZoneId') data = { 'region_id': region_obj.id, 'name': zone.get('LocalName'), 'instance_id': instance_id, } obj = ZoneModel.objects.filter(region_id=region_obj.id, instance_id=instance_id).first() # 存在就更新,不存在就创建 if obj: base_ctl.update_obj(ZoneModel, obj.id, data) else: data_list.append(data) data_list.append(data) base_ctl.create_objs(ZoneModel, data_list)
def sync_domain_records(): ''' 同步域名解析记录 ''' with transaction.atomic(): domain_objs = DomainModel.objects.all() old_ids = DomainRecordModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) existed_ids = [] record_list = [] key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunDNS(key, secret, 'cn-beijing') for domain_obj in domain_objs: page_num = 1 page_size = 50 while True: query = { 'domain_name': domain_obj.name, 'page_num': page_num, 'page_size': page_size, } data = ali_cli.get_domain_records(**query) total = data.get('total') data_list = data.get('data_list') for data in data_list: instance_id = data.get('RecordId') value = data.get('Value') rr = data.get('RR') typ = data.get('Type') enabled = True if data.get('Status') == 'ENABLE' else False query = { 'domain_id': domain_obj.id, 'instance_id': instance_id, } obj = DomainRecordModel.objects.filter(**query).first() data = query data['value'] = value data['rr'] = rr data['typ'] = typ # 这里name和fullname就是冗余存储,为了方便查询 data['name'] = domain_obj.name data['fullname'] = rr + '.' + domain_obj.name data['enabled'] = enabled if not obj: record_list.append(data) else: base_ctl.update_obj(DomainRecordModel, obj.id, data) existed_ids.append(obj.id) if total <= page_num * page_size: break page_num += 1 base_ctl.create_objs(DomainRecordModel, record_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) base_ctl.delete_objs(DomainRecordModel, deleted_ids) sync_domain_record_asset()
def sync_slb_backend_servers(): ''' 同步SLB默认服务器组服务器 ''' with transaction.atomic(): slb_objs = SlbModel.objects.all() query = { 'server_group__typ': SlbServerGroupModel.TYP_DEFAULT, } old_ids = SlbServerGroupEcsModel.objects.filter(**query).values_list( 'id', flat=True).all() old_ids = list(set(old_ids)) existed_ids = [] ecs_list = [] key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunSLB(key, secret, 'cn-beijing') for slb_obj in slb_objs: group_obj = SlbServerGroupModel.objects.filter(slb_id=slb_obj.id)\ .filter(typ=SlbServerGroupModel.TYP_DEFAULT).first() if not group_obj: data = { 'slb_id': slb_obj.id, 'name': 'default', 'instance_id': 'default', 'typ': SlbServerGroupModel.TYP_DEFAULT, } group_obj = base_ctl.create_obj(SlbServerGroupModel, data) ali_cli.reset_region(slb_obj.region_id) ecses = ali_cli.get_slb_info( slb_obj.instance_id).get('backend_servers') for ecs in ecses: ecs_instance_id = ecs.get('ServerId') ecs_obj = EcsModel.objects.filter( instance_id=ecs_instance_id).first() weight = ecs.get('Weight') obj = SlbServerGroupEcsModel.objects.filter(slb_id=slb_obj.id)\ .filter(server_group_id=group_obj.id, ecs_id=ecs_obj.id).first() data = { 'slb_id': slb_obj.id, 'server_group_id': group_obj.id, 'ecs_id': ecs_obj.id, 'weight': weight, } if not obj: ecs_list.append(data) else: base_ctl.update_obj(SlbServerGroupEcsModel, obj.id, data) existed_ids.append(obj.id) base_ctl.create_objs(SlbServerGroupEcsModel, ecs_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) if deleted_ids: base_ctl.delete_objs(SlbServerGroupEcsModel, deleted_ids) sync_slb_vserver_groups()
def sync_rds_databases(): ''' 同步RDS Database ''' with transaction.atomic(): rds_objs = RdsModel.objects.all() old_ids = RdsDatabaseModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) existed_ids = [] database_list = [] key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunRDS(key, secret, 'cn-beijing') for rds_obj in rds_objs: ali_cli.reset_region(rds_obj.region_id) page_num = 1 page_size = 50 while True: query = { 'page_num': page_num, 'page_size': page_size, 'instance_id': rds_obj.instance_id, } data = ali_cli.get_rds_databases(**query) data_list = data.get('data_list') for data in data_list: instance_id = data.get('DBName') name = data.get('DBName') desc = data.get('DBDescription') # 这里先存一下关联account信息,之后省得再用接口获取 accounts = data.get('Accounts').get('AccountPrivilegeInfo') query = { 'rds_id': rds_obj.id, 'instance_id': instance_id, } obj = RdsDatabaseModel.objects.filter(**query).first() data = query data['name'] = name data['desc'] = desc data['accounts'] = json.dumps(accounts) if not obj: database_list.append(data) else: base_ctl.update_obj(RdsDatabaseModel, obj.id, data) existed_ids.append(obj.id) if page_size > len(data_list): break page_num += 1 base_ctl.create_objs(RdsDatabaseModel, database_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) base_ctl.delete_objs(RdsDatabaseModel, deleted_ids) sync_rds_databases_accounts()
def sync_rdses(): ''' 同步RDS ''' with transaction.atomic(): key, secret = aliyun_key_ctl.get_enabled_aliyun_key() regions = region_ctl.get_regions( status=RegionModel.ST_ENABLE)['data_list'] # 记录原来已经创建过的RDS,用于之后删除已经不存在的使用 old_ids = RdsModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) # 用来存储仍然可以查到的RDS existed_ids = [] # 记录需要新创建的RDS信息,用于批量创建 rds_list = [] # 每次使用都先使用默认的地域初始化,其实可以在类里增加默认值,但是没有增加默认值是为了更明确知道在干什么 ali_cli = AliyunRDS(key, secret, 'cn-beijing') for region in regions: region_id = region.get('instance_id') ali_cli.reset_region(region_id) page_num = 1 page_size = 50 while True: query = { 'page_num': page_num, 'page_size': page_size, } data = ali_cli.get_rdses(**query) total = data.get('total') data_list = data.get('data_list') for data in data_list: data = format_rds_data(data) instance_id = data.get('instance_id') attribute = ali_cli.get_rds_attribute(instance_id) if attribute: data['connection'] = attribute.get('ConnectionString') obj = RdsModel.objects.filter( instance_id=instance_id).first() if obj: base_ctl.update_obj(RdsModel, obj.id, data) existed_ids.append(obj.id) else: rds_list.append(data) if total <= page_num * page_size: break page_num += 1 base_ctl.create_objs(RdsModel, rds_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) base_ctl.delete_objs(RdsModel, deleted_ids) sync_rds_accounts()
def sync_slb_vserver_group_backend_servers(): ''' 同步虚拟服务器组后端服务器 ''' with transaction.atomic(): query = { 'typ': SlbServerGroupModel.TYP_VSERVER, } group_objs = SlbServerGroupModel.objects.filter(**query).all() query = { 'server_group__typ': SlbServerGroupModel.TYP_VSERVER, } old_ids = SlbServerGroupEcsModel.objects.filter(**query).values_list( 'id', flat=True).all() old_ids = list(set(old_ids)) existed_ids = [] ecs_list = [] key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunSLB(key, secret, 'cn-beijing') for group_obj in group_objs: ali_cli.reset_region(group_obj.slb.region_id) ecses = ali_cli.get_vserver_group_backend_servers( group_obj.instance_id).get('data_list') for ecs in ecses: ecs_instance_id = ecs.get('ServerId') weight = ecs.get('Weight') ecs_obj = EcsModel.objects.filter( instance_id=ecs_instance_id).first() query = { 'slb_id': group_obj.slb_id, 'server_group_id': group_obj.id, 'ecs_id': ecs_obj.id, } obj = SlbServerGroupEcsModel.objects.filter(**query).first() data = { 'slb_id': group_obj.slb_id, 'server_group_id': group_obj.id, 'ecs_id': ecs_obj.id, 'weight': weight, } if not obj: ecs_list.append(data) else: base_ctl.update_obj(SlbServerGroupEcsModel, obj.id, data) existed_ids.append(obj.id) base_ctl.create_objs(SlbServerGroupEcsModel, ecs_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) if deleted_ids: base_ctl.delete_objs(SlbServerGroupEcsModel, deleted_ids)
def sync_slbs(): ''' 同步SLB ''' with transaction.atomic(): key, secret = aliyun_key_ctl.get_enabled_aliyun_key() regions = region_ctl.get_regions( status=RegionModel.ST_ENABLE)['data_list'] # 记录原来已经创建过的SLB,用于之后删除已经不存在的使用 old_ids = SlbModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) # 用来存储仍然可以查到的SLB existed_ids = [] # 记录需要新创建的SLB信息,用于批量创建 slb_list = [] # 每次使用都先使用默认的地域初始化,其实可以在类里增加默认值,但是没有增加默认值是为了更明确知道在干什么 ali_cli = AliyunSLB(key, secret, 'cn-beijing') for region in regions: region_id = region.get('instance_id') ali_cli.reset_region(region_id) page_num = 1 page_size = 50 while True: query = { 'page_num': page_num, 'page_size': page_size, } data = ali_cli.get_slbs(**query) total = data.get('total') data_list = data.get('data_list') for data in data_list: data = format_slb_data(data) instance_id = data.get('instance_id') obj = SlbModel.objects.filter( instance_id=instance_id).first() if obj: base_ctl.update_obj(SlbModel, obj.id, data) existed_ids.append(obj.id) else: slb_list.append(data) if total <= page_num * page_size: break page_num += 1 base_ctl.create_objs(SlbModel, slb_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) if deleted_ids: base_ctl.delete_objs(SlbModel, deleted_ids) sync_slb_backend_servers()
def sync_rds_accounts(): ''' 同步RDS账号 ''' with transaction.atomic(): rds_objs = RdsModel.objects.all() old_ids = RdsAccountModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) existed_ids = [] account_list = [] key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunRDS(key, secret, 'cn-beijing') for rds_obj in rds_objs: ali_cli.reset_region(rds_obj.region_id) page_num = 1 page_size = 50 while True: query = { 'page_num': page_num, 'page_size': page_size, 'instance_id': rds_obj.instance_id, } data = ali_cli.get_rds_accounts(**query) data_list = data.get('data_list') for data in data_list: username = data.get('AccountName') query = { 'rds_id': rds_obj.id, 'username': username, } obj = RdsAccountModel.objects.filter(**query).first() data = query if not obj: account_list.append(data) else: base_ctl.update_obj(RdsAccountModel, obj.id, data) existed_ids.append(obj.id) if page_size > len(data_list): break page_num += 1 base_ctl.create_objs(RdsAccountModel, account_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) base_ctl.delete_objs(RdsAccountModel, deleted_ids) sync_rds_databases()
def sync_slb_vserver_groups(): ''' 同步SLB虚拟服务器组 ''' with transaction.atomic(): slb_objs = SlbModel.objects.all() old_ids = SlbServerGroupModel.objects.filter(typ=SlbServerGroupModel.TYP_VSERVER)\ .values_list('id', flat=True).all() old_ids = list(set(old_ids)) existed_ids = [] group_list = [] key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunSLB(key, secret, 'cn-beijing') for slb_obj in slb_objs: ali_cli.reset_region(slb_obj.region_id) groups = ali_cli.get_vserver_groups( slb_obj.instance_id).get('data_list') for group in groups: group_instance_id = group.get('VServerGroupId') group_name = group.get('VServerGroupName') query = { 'slb_id': slb_obj.id, 'typ': SlbServerGroupModel.TYP_VSERVER, 'instance_id': group_instance_id, } group_obj = SlbServerGroupModel.objects.filter(**query).first() data = { 'slb_id': slb_obj.id, 'instance_id': group_instance_id, 'name': group_name, 'typ': SlbServerGroupModel.TYP_VSERVER, } if not group_obj: group_list.append(data) else: group_obj = base_ctl.update_obj(SlbServerGroupModel, group_obj.id, data) existed_ids.append(group_obj.id) base_ctl.create_objs(SlbServerGroupModel, group_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) if deleted_ids: base_ctl.delete_objs(SlbServerGroupModel, deleted_ids) sync_slb_vserver_group_backend_servers()
def sync_redis_accounts(): ''' 同步Redis账号 ''' with transaction.atomic(): redis_objs = RedisModel.objects.all() old_ids = RedisAccountModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) existed_ids = [] account_list = [] key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunRedis(key, secret, 'cn-beijing') for redis_obj in redis_objs: ali_cli.reset_region(redis_obj.region_id) query = { 'instance_id': redis_obj.instance_id, } data = ali_cli.get_redis_accounts(**query) data_list = data.get('data_list') for data in data_list: username = data.get('AccountName') typ = data.get('AccountType') status = data.get('AccountStatus') privilege = data.get('DatabasePrivileges').get( 'DatabasePrivilege')[0].get('AccountPrivilege') query = { 'redis_id': redis_obj.id, 'username': username, } obj = RedisAccountModel.objects.filter(**query).first() data = query data['typ'] = typ data['status'] = status data['privilege'] = privilege if not obj: account_list.append(data) else: base_ctl.update_obj(RedisAccountModel, obj.id, data) existed_ids.append(obj.id) base_ctl.create_objs(RedisAccountModel, account_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) base_ctl.delete_objs(RedisAccountModel, deleted_ids)
def sync_domains(): ''' 同步Domain ''' with transaction.atomic(): key, secret = aliyun_key_ctl.get_enabled_aliyun_key() # 记录原来已经创建过的Domain,用于之后删除已经不存在的使用 old_ids = DomainModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) # 用来存储仍然可以查到的Domain existed_ids = [] # 记录需要新创建的Domain信息,用于批量创建 domain_list = [] # 每次使用都先使用默认的地域初始化,其实可以在类里增加默认值,但是没有增加默认值是为了更明确知道在干什么 ali_cli = AliyunDNS(key, secret, 'cn-beijing') page_num = 1 page_size = 50 while True: query = { 'page_num': page_num, 'page_size': page_size, } data = ali_cli.get_domains(**query) total = data.get('total') data_list = data.get('data_list') for data in data_list: data = format_domain_data(data) instance_id = data.get('instance_id') obj = DomainModel.objects.filter(instance_id=instance_id).first() if obj: base_ctl.update_obj(DomainModel, obj.id, data) existed_ids.append(obj.id) else: domain_list.append(data) if total <= page_num * page_size: break page_num += 1 base_ctl.create_objs(DomainModel, domain_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) base_ctl.delete_objs(DomainModel, deleted_ids) sync_domain_records()
def sync_mongo_accounts(): ''' 同步Mongo账号 ''' with transaction.atomic(): mongo_objs = MongoModel.objects.all() old_ids = MongoAccountModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) existed_ids = [] account_list = [] key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunMongo(key, secret, 'cn-beijing') for mongo_obj in mongo_objs: ali_cli.reset_region(mongo_obj.region_id) query = { 'instance_id': mongo_obj.instance_id, } data = ali_cli.get_mongo_accounts(**query) data_list = data.get('data_list') for data in data_list: username = data.get('AccountName') typ = data.get('CharacterType') status = data.get('AccountStatus') remark = data.get('AccountDescription') query = { 'mongo_id': mongo_obj.id, 'username': username, } obj = MongoAccountModel.objects.filter(**query).first() data = query data['typ'] = typ data['status'] = status data['remark'] = remark if not obj: account_list.append(data) else: base_ctl.update_obj(MongoAccountModel, obj.id, data) existed_ids.append(obj.id) base_ctl.create_objs(MongoAccountModel, account_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) base_ctl.delete_objs(MongoAccountModel, deleted_ids)
def sync_rocket_groups(): ''' 同步Rocket Group ''' with transaction.atomic(): rocket_objs = RocketModel.objects.all() old_ids = RocketGroupModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) existed_ids = [] group_list = [] key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunONS(key, secret, 'cn-beijing') for rocket_obj in rocket_objs: ali_cli.reset_region(rocket_obj.region_id) query = { 'instance_id': rocket_obj.instance_id, } data = ali_cli.get_rocket_groups(**query) data_list = data.get('data_list') for data in data_list: name = data.get('GroupId') typ = data.get('GroupType') remark = data.get('Remark') query = { 'rocket_id': rocket_obj.id, 'name': name, } obj = RocketGroupModel.objects.filter(**query).first() data = query data['typ'] = typ data['remark'] = remark if not obj: group_list.append(data) else: base_ctl.update_obj(RocketGroupModel, obj.id, data) existed_ids.append(obj.id) base_ctl.create_objs(RocketGroupModel, group_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) base_ctl.delete_objs(RocketGroupModel, deleted_ids)
def sync_regions(): ''' 同步地域 ''' key, secret = aliyun_key_ctl.get_enabled_aliyun_key() ali_cli = AliyunManager(key, secret, 'cn-beijing') regions = ali_cli.get_regions()['data_list'] data_list = [] for region in regions: instance_id = region.get('RegionId') data = { 'name': region.get('LocalName'), 'instance_id': instance_id, 'endpoint': region.get('RegionEndpoint'), } obj = RegionModel.objects.filter(instance_id=instance_id).first() # 存在就更新,不存在就创建 if obj: base_ctl.update_obj(RegionModel, obj.id, data) else: data_list.append(data) base_ctl.create_objs(RegionModel, data_list)
def sync_rockets(): ''' 同步Rocket ''' with transaction.atomic(): key, secret = aliyun_key_ctl.get_enabled_aliyun_key() regions = region_ctl.get_regions( status=RegionModel.ST_ENABLE)['data_list'] # 记录原来已经创建过的Rocket,用于之后删除已经不存在的使用 old_ids = RocketModel.objects.values_list('id', flat=True).all() old_ids = list(set(old_ids)) # 用来存储仍然可以查到的Rocket existed_ids = [] # 记录需要新创建的Rocket信息,用于批量创建 rocket_list = [] # 每次使用都先使用默认的地域初始化,其实可以在类里增加默认值,但是没有增加默认值是为了更明确知道在干什么 ali_cli = AliyunONS(key, secret, 'cn-beijing') for region in regions: region_id = region.get('instance_id') ali_cli.reset_region(region_id) data = ali_cli.get_rockets() total = data.get('total') data_list = data.get('data_list') for data in data_list: data = format_rocket_data(data) data['region_id'] = region_id instance_id = data.get('instance_id') obj = RocketModel.objects.filter( instance_id=instance_id).first() if obj: base_ctl.update_obj(RocketModel, obj.id, data) existed_ids.append(obj.id) else: rocket_list.append(data) base_ctl.create_objs(RocketModel, rocket_list) deleted_ids = list(set(set(old_ids) - set(existed_ids))) base_ctl.delete_objs(RocketModel, deleted_ids) sync_rocket_topics()