def delete_service_asset_obj(obj_id,
environment_id,
typ,
asset_obj_id,
operator=None):
'''
删除服务关联资产实例
'''
# 零：判断typ值
if not ServiceAssetObjModel.check_choices('typ', typ):
raise errors.CommonError('资产实例类型值不正确')
# 一：判断服务是否关联对应资产模块
query = {
'service_id': obj_id,
'asset__sign': typ,
}
if not ServiceAssetModel.objects.filter(**query).exists():
raise errors.CommonError('服务需要先关联此资产模块')
# 二：判断资产实例是否存在
if not is_existed_asset_obj(typ, asset_obj_id):
raise errors.CommonError('资产实例不存在')
# 三：判断是否已经关联此资产实例
query = {
'service_id': obj_id,
'environment_id': environment_id,
'typ': typ,
'asset_obj_id': asset_obj_id,
}
obj = ServiceAssetObjModel.objects.filter(**query).first()
if not obj:
raise errors.CommonError('服务未关联此资产实例')
data = {
'status': ServiceAssetObjModel.ST_PENDING_REMOVE,
}
base_ctl.update_obj(ServiceAssetObjModel, obj.id, data, operator)
def sync_gitlabs():
'''
同步gitlab
这种同步，我认为没有必要加事务
'''
# 虽然我说我不建议有多个代码库，但是我写代码肯定是要支持多个的，因为并没有增加多少工作量
server_objs = GitlabServerModel.objects.all()
old_ids = GitlabProjectModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
project_list = []
for server_obj in server_objs:
gitlab_cli = GitlabCli(server_obj.host, server_obj.token)
projects = gitlab_cli.get_projects()
for project in projects:
project_id = project.id
query = {
'server_id': server_obj.id,
'project_id': project_id,
}
obj = GitlabProjectModel.objects.filter(**query).first()
data = query
data['name'] = project.path_with_namespace
data['web_url'] = project.web_url
data['ssh_url'] = project.ssh_url_to_repo
if obj:
base_ctl.update_obj(GitlabProjectModel, obj.id, data)
existed_ids.append(obj.id)
else:
project_list.append(data)
base_ctl.create_objs(GitlabProjectModel, project_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
if deleted_ids:
base_ctl.delete_objs(GitlabProjectModel, deleted_ids)
def sync_jenkins():
'''
同步jenkins
这种同步，我认为没有必要加事务
'''
server_objs = JenkinsServerModel.objects.all()
old_ids = JenkinsJobModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
job_list = []
for server_obj in server_objs:
jenkins_cli = JenkinsCli(server_obj.host, server_obj.username,
server_obj.token)
jobs = jenkins_cli.get_jobs()
for job in jobs:
job_name = job.get('name')
query = {
'server_id': server_obj.id,
'name': job_name,
}
obj = JenkinsJobModel.objects.filter(**query).first()
data = query
data['url'] = job.get('url')
if obj:
base_ctl.update_obj(JenkinsJobModel, obj.id, data)
existed_ids.append(obj.id)
else:
job_list.append(data)
base_ctl.create_objs(JenkinsJobModel, job_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
if deleted_ids:
base_ctl.delete_objs(JenkinsJobModel, deleted_ids)
def apply_task(berry_id):
try:
berry_obj = base_ctl.get_obj(BerryModel, berry_id)
# 一进来就更改任务状态到进行中
data = {
'status': BerryModel.ST_RUNNING,
}
base_ctl.update_obj(BerryModel, berry_id, data)
sync_list = [
'sync_ecs', 'sync_slb', 'sync_rds', 'sync_redis', 'sync_mongo',
'sync_rocket', 'sync_domain', 'sync_gitlab', 'sync_jenkins',
'sync_ldap_user'
]
# 如果是同步任务，则走此处理方式
if berry_obj.typ.sign in sync_list:
sync_task_route(berry_obj)
except Exception as e:
# 如果出现异常，则更改任务状态，并且记录错误日志
log = traceback.format_exc()
dt_end = time_utils.now()
duration = dt_end - berry_obj.dt_start
duration = int(duration.total_seconds())
data = {
'status': BerryModel.ST_FAILURE,
'error_log': log,
'dt_end': dt_end,
'duration': duration,
}
base_ctl.update_obj(BerryModel, berry_id, data)
def sync_zones():
'''
同步可用区
'''
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunManager(key, secret, 'cn-beijing')
region_objs = RegionModel.objects.all()
data_list = []
for region_obj in region_objs:
zones = ali_cli.get_zones(region_obj.instance_id)['data_list']
for zone in zones:
instance_id = zone.get('ZoneId')
data = {
'region_id': region_obj.id,
'name': zone.get('LocalName'),
'instance_id': instance_id,
}
obj = ZoneModel.objects.filter(region_id=region_obj.id, instance_id=instance_id).first()
# 存在就更新，不存在就创建
if obj:
base_ctl.update_obj(ZoneModel, obj.id, data)
else:
data_list.append(data)
data_list.append(data)
base_ctl.create_objs(ZoneModel, data_list)
def sync_domain_records():
'''
同步域名解析记录
'''
with transaction.atomic():
domain_objs = DomainModel.objects.all()
old_ids = DomainRecordModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
record_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunDNS(key, secret, 'cn-beijing')
for domain_obj in domain_objs:
page_num = 1
page_size = 50
while True:
query = {
'domain_name': domain_obj.name,
'page_num': page_num,
'page_size': page_size,
}
data = ali_cli.get_domain_records(**query)
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
instance_id = data.get('RecordId')
value = data.get('Value')
rr = data.get('RR')
typ = data.get('Type')
enabled = True if data.get('Status') == 'ENABLE' else False
query = {
'domain_id': domain_obj.id,
'instance_id': instance_id,
}
obj = DomainRecordModel.objects.filter(**query).first()
data = query
data['value'] = value
data['rr'] = rr
data['typ'] = typ
# 这里name和fullname就是冗余存储，为了方便查询
data['name'] = domain_obj.name
data['fullname'] = rr + '.' + domain_obj.name
data['enabled'] = enabled
if not obj:
record_list.append(data)
else:
base_ctl.update_obj(DomainRecordModel, obj.id, data)
existed_ids.append(obj.id)
if total <= page_num * page_size:
break
page_num += 1
base_ctl.create_objs(DomainRecordModel, record_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(DomainRecordModel, deleted_ids)
sync_domain_record_asset()
def sync_slb_backend_servers():
'''
同步SLB默认服务器组服务器
'''
with transaction.atomic():
slb_objs = SlbModel.objects.all()
query = {
'server_group__typ': SlbServerGroupModel.TYP_DEFAULT,
}
old_ids = SlbServerGroupEcsModel.objects.filter(**query).values_list(
'id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
ecs_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunSLB(key, secret, 'cn-beijing')
for slb_obj in slb_objs:
group_obj = SlbServerGroupModel.objects.filter(slb_id=slb_obj.id)\
.filter(typ=SlbServerGroupModel.TYP_DEFAULT).first()
if not group_obj:
data = {
'slb_id': slb_obj.id,
'name': 'default',
'instance_id': 'default',
'typ': SlbServerGroupModel.TYP_DEFAULT,
}
group_obj = base_ctl.create_obj(SlbServerGroupModel, data)
ali_cli.reset_region(slb_obj.region_id)
ecses = ali_cli.get_slb_info(
slb_obj.instance_id).get('backend_servers')
for ecs in ecses:
ecs_instance_id = ecs.get('ServerId')
ecs_obj = EcsModel.objects.filter(
instance_id=ecs_instance_id).first()
weight = ecs.get('Weight')
obj = SlbServerGroupEcsModel.objects.filter(slb_id=slb_obj.id)\
.filter(server_group_id=group_obj.id, ecs_id=ecs_obj.id).first()
data = {
'slb_id': slb_obj.id,
'server_group_id': group_obj.id,
'ecs_id': ecs_obj.id,
'weight': weight,
}
if not obj:
ecs_list.append(data)
else:
base_ctl.update_obj(SlbServerGroupEcsModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(SlbServerGroupEcsModel, ecs_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
if deleted_ids:
base_ctl.delete_objs(SlbServerGroupEcsModel, deleted_ids)
sync_slb_vserver_groups()
def create_berry(name,
typ,
time_mode=BerryModel.TIME_MODE_NOW,
dt_start=None,
input_params={},
operator=None):
'''
创建任务
params: dict
'''
query = {
'sign': typ,
}
typ_obj = BerryTypeModel.objects.filter(**query).first()
if not typ_obj:
raise errors.CommonError('任务类型不存在')
if not BerryModel.check_choices('time_mode', time_mode):
raise errors.CommonError('时间模式值不正确')
now = time_utils.now()
countdown = 0
if time_mode == BerryModel.TIME_MODE_CRONTAB:
if not dt_start:
raise errors.CommonError('定时任务必须传开始时间')
if dt_start <= now:
raise errors.CommonError('不能指定过去的时间')
countdown = dt_start - now
countdown = int(countdown.total_seconds())
if countdown < 5 * 60:
raise errors.CommonError('定时任务必须指定五分钟以后的时间')
else:
dt_start = time_utils.now()
user_id = None
if operator:
user_id = operator.id
data = {
'name': name,
'typ_id': typ_obj.id,
'time_mode': time_mode,
'dt_start': dt_start,
'input_params': json.dumps(input_params),
'user_id': user_id,
}
berry_obj = base_ctl.create_obj(BerryModel, data)
result = berry_tasks.apply_task.apply_async(countdown=countdown,
args=[berry_obj.id])
data = {
'task_id': result.task_id,
}
base_ctl.update_obj(BerryModel, berry_obj.id, data)
def sync_rds_databases():
'''
同步RDS Database
'''
with transaction.atomic():
rds_objs = RdsModel.objects.all()
old_ids = RdsDatabaseModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
database_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunRDS(key, secret, 'cn-beijing')
for rds_obj in rds_objs:
ali_cli.reset_region(rds_obj.region_id)
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
'instance_id': rds_obj.instance_id,
}
data = ali_cli.get_rds_databases(**query)
data_list = data.get('data_list')
for data in data_list:
instance_id = data.get('DBName')
name = data.get('DBName')
desc = data.get('DBDescription')
# 这里先存一下关联account信息，之后省得再用接口获取
accounts = data.get('Accounts').get('AccountPrivilegeInfo')
query = {
'rds_id': rds_obj.id,
'instance_id': instance_id,
}
obj = RdsDatabaseModel.objects.filter(**query).first()
data = query
data['name'] = name
data['desc'] = desc
data['accounts'] = json.dumps(accounts)
if not obj:
database_list.append(data)
else:
base_ctl.update_obj(RdsDatabaseModel, obj.id, data)
existed_ids.append(obj.id)
if page_size > len(data_list):
break
page_num += 1
base_ctl.create_objs(RdsDatabaseModel, database_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RdsDatabaseModel, deleted_ids)
sync_rds_databases_accounts()
def sync_rdses():
'''
同步RDS
'''
with transaction.atomic():
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
regions = region_ctl.get_regions(
status=RegionModel.ST_ENABLE)['data_list']
# 记录原来已经创建过的RDS，用于之后删除已经不存在的使用
old_ids = RdsModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
# 用来存储仍然可以查到的RDS
existed_ids = []
# 记录需要新创建的RDS信息，用于批量创建
rds_list = []
# 每次使用都先使用默认的地域初始化，其实可以在类里增加默认值，但是没有增加默认值是为了更明确知道在干什么
ali_cli = AliyunRDS(key, secret, 'cn-beijing')
for region in regions:
region_id = region.get('instance_id')
ali_cli.reset_region(region_id)
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
}
data = ali_cli.get_rdses(**query)
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
data = format_rds_data(data)
instance_id = data.get('instance_id')
attribute = ali_cli.get_rds_attribute(instance_id)
if attribute:
data['connection'] = attribute.get('ConnectionString')
obj = RdsModel.objects.filter(
instance_id=instance_id).first()
if obj:
base_ctl.update_obj(RdsModel, obj.id, data)
existed_ids.append(obj.id)
else:
rds_list.append(data)
if total <= page_num * page_size:
break
page_num += 1
base_ctl.create_objs(RdsModel, rds_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RdsModel, deleted_ids)
sync_rds_accounts()
def sync_slb_vserver_group_backend_servers():
'''
同步虚拟服务器组后端服务器
'''
with transaction.atomic():
query = {
'typ': SlbServerGroupModel.TYP_VSERVER,
}
group_objs = SlbServerGroupModel.objects.filter(**query).all()
query = {
'server_group__typ': SlbServerGroupModel.TYP_VSERVER,
}
old_ids = SlbServerGroupEcsModel.objects.filter(**query).values_list(
'id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
ecs_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunSLB(key, secret, 'cn-beijing')
for group_obj in group_objs:
ali_cli.reset_region(group_obj.slb.region_id)
ecses = ali_cli.get_vserver_group_backend_servers(
group_obj.instance_id).get('data_list')
for ecs in ecses:
ecs_instance_id = ecs.get('ServerId')
weight = ecs.get('Weight')
ecs_obj = EcsModel.objects.filter(
instance_id=ecs_instance_id).first()
query = {
'slb_id': group_obj.slb_id,
'server_group_id': group_obj.id,
'ecs_id': ecs_obj.id,
}
obj = SlbServerGroupEcsModel.objects.filter(**query).first()
data = {
'slb_id': group_obj.slb_id,
'server_group_id': group_obj.id,
'ecs_id': ecs_obj.id,
'weight': weight,
}
if not obj:
ecs_list.append(data)
else:
base_ctl.update_obj(SlbServerGroupEcsModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(SlbServerGroupEcsModel, ecs_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
if deleted_ids:
base_ctl.delete_objs(SlbServerGroupEcsModel, deleted_ids)
def sync_slbs():
'''
同步SLB
'''
with transaction.atomic():
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
regions = region_ctl.get_regions(
status=RegionModel.ST_ENABLE)['data_list']
# 记录原来已经创建过的SLB，用于之后删除已经不存在的使用
old_ids = SlbModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
# 用来存储仍然可以查到的SLB
existed_ids = []
# 记录需要新创建的SLB信息，用于批量创建
slb_list = []
# 每次使用都先使用默认的地域初始化，其实可以在类里增加默认值，但是没有增加默认值是为了更明确知道在干什么
ali_cli = AliyunSLB(key, secret, 'cn-beijing')
for region in regions:
region_id = region.get('instance_id')
ali_cli.reset_region(region_id)
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
}
data = ali_cli.get_slbs(**query)
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
data = format_slb_data(data)
instance_id = data.get('instance_id')
obj = SlbModel.objects.filter(
instance_id=instance_id).first()
if obj:
base_ctl.update_obj(SlbModel, obj.id, data)
existed_ids.append(obj.id)
else:
slb_list.append(data)
if total <= page_num * page_size:
break
page_num += 1
base_ctl.create_objs(SlbModel, slb_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
if deleted_ids:
base_ctl.delete_objs(SlbModel, deleted_ids)
sync_slb_backend_servers()
def set_aliyun_key_status(obj_id, status, operator):
'''
设置阿里云key状态
'''
if not AliyunKeyModel.check_choices('status', status):
raise errors.CommonError('状态值不正确')
if status != AliyunKeyModel.ST_ENABLE:
raise errors.CommonError('暂时只支持启用操作')
obj = base_ctl.get_obj(AliyunKeyModel, obj_id)
if obj.status == status:
return
with transaction.atomic():
# 先禁用所有启用状态
existed_ids = AliyunKeyModel.objects.filter(status=AliyunKeyModel.ST_ENABLE)\
.values_list('id', flat=True).all()
data = {
'status': AliyunKeyModel.ST_DISABLE,
}
base_ctl.update_objs(AliyunKeyModel, existed_ids, data)
# 再启用
data = {
'status': status,
}
obj = base_ctl.update_obj(AliyunKeyModel, obj_id, data, operator)
data = obj.to_dict()
return data
def update_user(obj_id,
name=None,
password=None,
phone=None,
email=None,
operator=None):
'''
编辑用户
'''
obj = base_ctl.get_obj(UserModel, obj_id)
if not obj:
raise errors.CommonError('用户不存在')
if is_admin(obj):
raise errors.CommonError('超级管理员用户不允许编辑')
data = {
'name': name,
'phone': phone,
'email': email,
}
with transaction.atomic():
user_obj = base_ctl.update_obj(UserModel, obj_id, data, operator)
if password:
user_obj.set_password(password)
data = user_obj.to_dict()
return data
def update_jenkins_server(obj_id,
name,
host,
username,
password,
token,
operator=None):
'''
编辑Jenkins服务
'''
if JenkinsServerModel.objects.filter(name=name).exclude(
id=obj_id).exists():
raise errors.CommonError('Jenkins已经存在')
if JenkinsServerModel.objects.filter(host=host).exclude(
id=obj_id).exists():
raise errors.CommonError('Jenkins已经存在')
data = {
'name': name,
'host': host,
'username': username,
'password': password,
'token': token,
}
obj = base_ctl.update_obj(JenkinsServerModel, obj_id, data, operator)
data = obj.to_dict()
return data
def sync_rds_databases_accounts():
'''
同步Database关联Account
'''
with transaction.atomic():
rds_objs = RdsModel.objects.all()
old_ids = RdsDatabaseAccountModel.objects.values_list('id',
flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
related_list = []
for rds_obj in rds_objs:
# 一次取出同一个RDS下所有账号，并形成以username为key、id为value的字典
accounts = RdsAccountModel.objects.filter(
rds_id=rds_obj.id).values_list('username', 'id').all()
all_account_dict = dict(accounts)
database_objs = RdsDatabaseModel.objects.filter(
rds_id=rds_obj.id).all()
for database_obj in database_objs:
accounts = json.loads(database_obj.accounts)
for account in accounts:
username = account.get('Account')
privilege = account.get('AccountPrivilege')
# 如果username不存在，则说明是同步之前加的就不在此次处理范围
if username not in all_account_dict.keys():
continue
account_id = all_account_dict[username]
query = {
'database_id': database_obj.id,
'account_id': account_id,
}
obj = RdsDatabaseAccountModel.objects.filter(
**query).first()
data = query
data['privilege'] = privilege
if not obj:
related_list.append(data)
else:
base_ctl.update_obj(RdsDatabaseAccountModel, obj.id,
data)
existed_ids.append(obj.id)
base_ctl.create_objs(RdsDatabaseAccountModel, related_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RdsDatabaseAccountModel, deleted_ids)
def sync_rds_accounts():
'''
同步RDS账号
'''
with transaction.atomic():
rds_objs = RdsModel.objects.all()
old_ids = RdsAccountModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
account_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunRDS(key, secret, 'cn-beijing')
for rds_obj in rds_objs:
ali_cli.reset_region(rds_obj.region_id)
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
'instance_id': rds_obj.instance_id,
}
data = ali_cli.get_rds_accounts(**query)
data_list = data.get('data_list')
for data in data_list:
username = data.get('AccountName')
query = {
'rds_id': rds_obj.id,
'username': username,
}
obj = RdsAccountModel.objects.filter(**query).first()
data = query
if not obj:
account_list.append(data)
else:
base_ctl.update_obj(RdsAccountModel, obj.id, data)
existed_ids.append(obj.id)
if page_size > len(data_list):
break
page_num += 1
base_ctl.create_objs(RdsAccountModel, account_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RdsAccountModel, deleted_ids)
sync_rds_databases()
def update_account(obj_id, password, operator=None):
'''
编辑账号
'''
data = {
'password': password,
}
obj = base_ctl.update_obj(RdsAccountModel, obj_id, data, operator)
data = obj.to_dict()
return data
def sync_redis_accounts():
'''
同步Redis账号
'''
with transaction.atomic():
redis_objs = RedisModel.objects.all()
old_ids = RedisAccountModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
account_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunRedis(key, secret, 'cn-beijing')
for redis_obj in redis_objs:
ali_cli.reset_region(redis_obj.region_id)
query = {
'instance_id': redis_obj.instance_id,
}
data = ali_cli.get_redis_accounts(**query)
data_list = data.get('data_list')
for data in data_list:
username = data.get('AccountName')
typ = data.get('AccountType')
status = data.get('AccountStatus')
privilege = data.get('DatabasePrivileges').get(
'DatabasePrivilege')[0].get('AccountPrivilege')
query = {
'redis_id': redis_obj.id,
'username': username,
}
obj = RedisAccountModel.objects.filter(**query).first()
data = query
data['typ'] = typ
data['status'] = status
data['privilege'] = privilege
if not obj:
account_list.append(data)
else:
base_ctl.update_obj(RedisAccountModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(RedisAccountModel, account_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RedisAccountModel, deleted_ids)
def sync_domains():
'''
同步Domain
'''
with transaction.atomic():
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
# 记录原来已经创建过的Domain，用于之后删除已经不存在的使用
old_ids = DomainModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
# 用来存储仍然可以查到的Domain
existed_ids = []
# 记录需要新创建的Domain信息，用于批量创建
domain_list = []
# 每次使用都先使用默认的地域初始化，其实可以在类里增加默认值，但是没有增加默认值是为了更明确知道在干什么
ali_cli = AliyunDNS(key, secret, 'cn-beijing')
page_num = 1
page_size = 50
while True:
query = {
'page_num': page_num,
'page_size': page_size,
}
data = ali_cli.get_domains(**query)
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
data = format_domain_data(data)
instance_id = data.get('instance_id')
obj = DomainModel.objects.filter(instance_id=instance_id).first()
if obj:
base_ctl.update_obj(DomainModel, obj.id, data)
existed_ids.append(obj.id)
else:
domain_list.append(data)
if total <= page_num * page_size:
break
page_num += 1
base_ctl.create_objs(DomainModel, domain_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(DomainModel, deleted_ids)
sync_domain_records()
def sync_mongo_accounts():
'''
同步Mongo账号
'''
with transaction.atomic():
mongo_objs = MongoModel.objects.all()
old_ids = MongoAccountModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
account_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunMongo(key, secret, 'cn-beijing')
for mongo_obj in mongo_objs:
ali_cli.reset_region(mongo_obj.region_id)
query = {
'instance_id': mongo_obj.instance_id,
}
data = ali_cli.get_mongo_accounts(**query)
data_list = data.get('data_list')
for data in data_list:
username = data.get('AccountName')
typ = data.get('CharacterType')
status = data.get('AccountStatus')
remark = data.get('AccountDescription')
query = {
'mongo_id': mongo_obj.id,
'username': username,
}
obj = MongoAccountModel.objects.filter(**query).first()
data = query
data['typ'] = typ
data['status'] = status
data['remark'] = remark
if not obj:
account_list.append(data)
else:
base_ctl.update_obj(MongoAccountModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(MongoAccountModel, account_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(MongoAccountModel, deleted_ids)
def set_region_status(obj_id, status, operator):
'''
设置地域状态
'''
if not RegionModel.check_choices('status', status):
raise errors.CommonError('状态值不正确')
data = {
'status': status,
}
obj = base_ctl.update_obj(RegionModel, obj_id, data)
data = obj.to_dict()
return data
def sync_regions():
'''
同步地域
'''
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunManager(key, secret, 'cn-beijing')
regions = ali_cli.get_regions()['data_list']
data_list = []
for region in regions:
instance_id = region.get('RegionId')
data = {
'name': region.get('LocalName'),
'instance_id': instance_id,
'endpoint': region.get('RegionEndpoint'),
}
obj = RegionModel.objects.filter(instance_id=instance_id).first()
# 存在就更新，不存在就创建
if obj:
base_ctl.update_obj(RegionModel, obj.id, data)
else:
data_list.append(data)
base_ctl.create_objs(RegionModel, data_list)
def sync_rocket_groups():
'''
同步Rocket Group
'''
with transaction.atomic():
rocket_objs = RocketModel.objects.all()
old_ids = RocketGroupModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
existed_ids = []
group_list = []
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
ali_cli = AliyunONS(key, secret, 'cn-beijing')
for rocket_obj in rocket_objs:
ali_cli.reset_region(rocket_obj.region_id)
query = {
'instance_id': rocket_obj.instance_id,
}
data = ali_cli.get_rocket_groups(**query)
data_list = data.get('data_list')
for data in data_list:
name = data.get('GroupId')
typ = data.get('GroupType')
remark = data.get('Remark')
query = {
'rocket_id': rocket_obj.id,
'name': name,
}
obj = RocketGroupModel.objects.filter(**query).first()
data = query
data['typ'] = typ
data['remark'] = remark
if not obj:
group_list.append(data)
else:
base_ctl.update_obj(RocketGroupModel, obj.id, data)
existed_ids.append(obj.id)
base_ctl.create_objs(RocketGroupModel, group_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RocketGroupModel, deleted_ids)
def update_aliyun_key(obj_id, key, secret, operator):
'''
编辑阿里云key
'''
if AliyunKeyModel.objects.filter(key=key).exclude(id=obj_id).exists():
raise errors.CommonError('此Key已经存在')
data = {
'key': key,
'secret': secret,
}
obj = base_ctl.update_obj(AliyunKeyModel, obj_id, data, operator)
data = obj.to_dict()
return data
def sync_rockets():
'''
同步Rocket
'''
with transaction.atomic():
key, secret = aliyun_key_ctl.get_enabled_aliyun_key()
regions = region_ctl.get_regions(
status=RegionModel.ST_ENABLE)['data_list']
# 记录原来已经创建过的Rocket，用于之后删除已经不存在的使用
old_ids = RocketModel.objects.values_list('id', flat=True).all()
old_ids = list(set(old_ids))
# 用来存储仍然可以查到的Rocket
existed_ids = []
# 记录需要新创建的Rocket信息，用于批量创建
rocket_list = []
# 每次使用都先使用默认的地域初始化，其实可以在类里增加默认值，但是没有增加默认值是为了更明确知道在干什么
ali_cli = AliyunONS(key, secret, 'cn-beijing')
for region in regions:
region_id = region.get('instance_id')
ali_cli.reset_region(region_id)
data = ali_cli.get_rockets()
total = data.get('total')
data_list = data.get('data_list')
for data in data_list:
data = format_rocket_data(data)
data['region_id'] = region_id
instance_id = data.get('instance_id')
obj = RocketModel.objects.filter(
instance_id=instance_id).first()
if obj:
base_ctl.update_obj(RocketModel, obj.id, data)
existed_ids.append(obj.id)
else:
rocket_list.append(data)
base_ctl.create_objs(RocketModel, rocket_list)
deleted_ids = list(set(set(old_ids) - set(existed_ids)))
base_ctl.delete_objs(RocketModel, deleted_ids)
sync_rocket_topics()
def sync_task_route(berry_obj):
'''
同步任务
'''
if berry_obj.typ.sign == 'sync_ecs':
ecs_sync.sync_ecses()
elif berry_obj.typ.sign == 'sync_slb':
slb_sync.sync_slbs()
elif berry_obj.typ.sign == 'sync_rds':
rds_sync.sync_rdses()
elif berry_obj.typ.sign == 'sync_redis':
redis_sync.sync_redises()
elif berry_obj.typ.sign == 'sync_mongo':
mongo_sync.sync_mongos()
elif berry_obj.typ.sign == 'sync_domain':
domain_sync.sync_domains()
elif berry_obj.typ.sign == 'sync_rocket':
rocket_sync.sync_rockets()
elif berry_obj.typ.sign == 'sync_gitlab':
gitlab_sync.sync_gitlabs()
elif berry_obj.typ.sign == 'sync_jenkins':
jenkins_sync.sync_jenkins()
elif berry_obj.typ.sign == 'sync_ldap_user':
account_sync.sync_ldap_user()
else:
raise errors.CommonError(
f'任务{berry_obj.id}: 不存在的任务类型{berry_obj.typ.sign}')
# 执行成功
dt_end = time_utils.now()
duration = dt_end - berry_obj.dt_start
duration = int(duration.total_seconds())
data = {
'status': BerryModel.ST_SUCCESS,
'dt_end': dt_end,
'duration': duration,
}
base_ctl.update_obj(BerryModel, berry_obj.id, data)
def update_asset(obj_id, name, password, remark, operator):
'''
编辑资产
'''
obj = base_ctl.get_obj(AssetModel, obj_id)
if obj.user_id != operator.id:
raise errors.CommonError('数据异常')
data = {
'name': name,
'password': password,
'remark': remark,
}
obj = base_ctl.update_obj(AssetModel, obj_id, data, operator)
data = obj.to_dict()
return data
def update_department(obj_id, name, sign, operator=None):
'''
编辑部门
'''
if DepartmentModel.objects.filter(name=name).exclude(id=obj_id).exists():
raise errors.CommonError('部门名称已存在')
if DepartmentModel.objects.filter(sign=sign).exclude(id=obj_id).exists():
raise errors.CommonError('部门标识已存在')
obj = base_ctl.get_obj(DepartmentModel, obj_id)
data = {
'name': name,
'sign': sign,
}
obj = base_ctl.update_obj(DepartmentModel, obj_id, data, operator)
data = obj.to_dict()
return data
def update_frame(obj_id, name, sign, language_id, operator=None):
'''
编辑框架
'''
if FrameModel.objects.filter(name=name).exclude(id=obj_id).exists():
raise errors.CommonError('框架名称已存在')
if FrameModel.objects.filter(sign=sign).exclude(id=obj_id).exists():
raise errors.CommonError('框架标识已存在')
data = {
'name': name,
'sign': sign,
'language_id': language_id,
}
obj = base_ctl.update_obj(FrameModel, obj_id, data, operator)
data = obj.to_dict()
return data
