def post(self):
argus = self.body_arguments
publish_plan_id = argus.pop('publish_plan_id', None)
if not publish_plan_id:
raise HTTPError(status_code=400,
reason='Missing arguments: publish_plan_id')
res = []
with session_scope() as ss:
publish_plan = ss.query(PublishPlan).filter_by(
id=publish_plan_id).one_or_none()
if not publish_plan or publish_plan.status != 21:
raise HTTPError(
status_code=400,
reason=
' publish_plan_id {} is not exits or not in rollback status'
.format(publish_plan_id))
retry_pattern_id_list = get_retry_rollback_pattern_ids(publish_plan_id)
if not retry_pattern_id_list:
raise HTTPError(status_code=400,
reason="Has no retry_patterns to retry")
ready_run_tasks = run_rollback_pattern(retry_pattern_id_list)
with session_scope() as ss:
db_res = ss.query(PublishPattern).filter(
PublishPattern.id.in_(retry_pattern_id_list)).all()
for pattern in db_res:
r = pattern.to_dict()
r['publish_application_hosts'] = get_publish_hosts(pattern)
res.append(r)
res.append({'ready_run_task': ready_run_tasks})
self.render_json_response(code=200, msg="OK", res=res)
def post(self):
argus = self.body_arguments
publish_plan_id = argus.pop('publish_plan_id', None)
if not publish_plan_id:
raise HTTPError(status_code=400,
reason='Missing arguments: publish_plan_id')
res = []
host_application_list = get_host_by_publish_plan_id(publish_plan_id)
if not host_application_list:
raise HTTPError(
status_code=400,
reason="publish_plan_id {} has no finished pattern".format(
publish_plan_id))
all_pattern_id_list = generate_rollback_steps(publish_plan_id,
host_application_list)
with session_scope() as ss:
# update publish plan status
ss.query(PublishPlan).filter_by(id=publish_plan_id).update(
{"status": 21})
# 把QA测试的步骤去掉
p_res = ss.query(PublishPattern).filter(
PublishPattern.action > 10, PublishPattern.action != 13,
PublishPattern.id.in_(all_pattern_id_list)).order_by(
PublishPattern.step).all()
pattern_id_list = []
for pattern in p_res:
# 有停顿， 不再往下
if pattern.action == 19:
break
pattern_id_list.append(pattern.id)
for pattern_id in pattern_id_list:
audit_log(handler=self,
description="执行回滚步骤",
resource_type=4,
resource_id=pattern_id)
ready_run_tasks = run_rollback_pattern(pattern_id_list)
with session_scope() as ss:
db_res = ss.query(PublishPattern).filter(
PublishPattern.id.in_(pattern_id_list)).all()
for pattern in db_res:
r = pattern.to_dict()
r['publish_application_hosts'] = get_publish_hosts(pattern)
res.append(r)
res.append({'ready_run_task': ready_run_tasks})
self.render_json_response(code=200, msg="OK", res=res)
def get(self, *args, **kwargs):
"""
cmd： status/enable/disable
"""
argus = self.url_arguments
pattern_id = argus.pop('pattern_id', None)
cmdstr = argus.pop('cmd', 'status')
project_name = argus.pop('application_name', None)
publish_host_ids = argus.pop('publish_host_ids', '')
if not publish_host_ids or not cmdstr:
raise HTTPError(
status=400,
reason="Missing arguments publish_host_ids or cmdstr")
publish_host_id_list = publish_host_ids.split(',')
task_name = task_name_map[cmdstr]
with session_scope() as ss:
host_and_id_list = run_celery_task(
session=ss,
publish_host_id_list=publish_host_id_list,
task_name=task_name,
project_name=project_name,
pattern_id=pattern_id)
for resource_id in publish_host_id_list:
audit_log(self,
description=task_name,
resource_type=3,
resource_id=resource_id)
self.render_json_response(code=200, msg="OK", res=host_and_id_list)
def put(self):
res = {}
rightful_keys = (
'id',
'step',
'title',
'note',
'action',
'status',
)
body_kwargs = take_out_unrightful_arguments(rightful_keys, self.body_arguments)
id_ = body_kwargs.pop('id', None)
if id_ is None:
raise HTTPError(status_code=400, reason='Missing argument "id"')
with session_scope() as ss:
q = ss.query(PublishPattern).filter_by(id=id_)
q.update(body_kwargs)
db_res = q.one_or_none()
if db_res is not None:
res = db_res.to_dict()
audit_log(self, description='更新发版步骤', resource_type=4, resource_id=id_)
self.render_json_response(code=200, res=res)
def post(self, *args, **kwargs):
argus = self.body_arguments
if not isinstance(argus, list):
raise HTTPError(status_code=400, reason='arguments must be list')
res = []
with session_scope() as ss:
for item in argus:
pattern_id = item.pop('pattern_id', None)
version = item['version']
project_name = item.pop('application_name', None)
publish_host_ids = item.pop('publish_host_ids', None)
if not (project_name or version or publish_host_ids
or pattern_id):
raise HTTPError(status_code=400,
reason="Missing arguments")
task_name = '执行发版'
host_and_id_list = run_celery_task(
session=ss,
publish_host_id_list=publish_host_ids,
task_name=task_name,
pattern_id=pattern_id,
version=version,
project_name=project_name)
res.extend(host_and_id_list)
for resource_id in publish_host_ids:
audit_log(self,
description=task_name,
resource_type=3,
resource_id=resource_id)
self.render_json_response(code=200, msg="OK", res=res)
def on_failure(self, exc, task_id, args, kwargs, einfo):
logger.error('{0!r} failed: {1!r}'.format(task_id, exc))
argus = {
"status": "FAILED",
"result": str(exc),
"end_time": datetime.now()
}
with session_scope() as ss:
publish_task = ss.query(PublishTask).filter(
PublishTask.celery_task_id == task_id).one_or_none()
if publish_task is None:
logger.error('PublishTask {} is not exist!'.format(task_id))
else:
for k, v in argus.items():
publish_task.__setattr__(k, v)
ss.flush()
pattern_task_id = publish_task.publish_pattern_task_id
if pattern_task_id != 0:
pattern_task = upgrade_pattern_task_status(
ss, pattern_task_id, argus['status'])
pattern_host_id = pattern_task.publish_pattern_host_id
pattern_hosts = upgrade_pattern_host_status(
ss, pattern_host_id)
pattern_id = pattern_hosts[0].publish_pattern_id
upgrade_pattern_status(ss, pattern_id)
def do_success(self, retval, task_id, args, kwargs):
argus = {
"status": "SUCCESS",
"result": json.dumps(retval) if retval else None,
"end_time": datetime.now()
}
with session_scope() as ss:
publish_task = ss.query(PublishTask).filter(
PublishTask.celery_task_id == task_id).one_or_none()
if publish_task is None:
raise TaskError('PublishTask {} is not exist!'.format(task_id))
else:
for k, v in argus.items():
publish_task.__setattr__(k, v)
ss.flush()
pattern_task_id = publish_task.publish_pattern_task_id
if pattern_task_id != 0:
pattern_task = upgrade_pattern_task_status(
ss, pattern_task_id, argus['status'])
pattern_host_id = pattern_task.publish_pattern_host_id
pattern_hosts = upgrade_pattern_host_status(
ss, pattern_host_id)
pattern_id = pattern_hosts[0].publish_pattern_id
upgrade_pattern_status(ss, pattern_id)
return True
def do_success(self, retval, task_id, args, kwargs):
with session_scope() as ss:
publish_task = ss.query(PublishTask).filter(
PublishTask.celery_task_id == task_id).one_or_none()
if publish_task is None:
raise TaskError('PublishTask {} is not exist!'.format(task_id))
else:
ansible_result = self.get_ansible_result(retval)
if ansible_result.get('rc') == 0:
ansible_status = 'SUCCESS'
else:
ansible_status = 'FAILED'
publish_task.status = ansible_status
publish_task.end_time = datetime.now()
try:
publish_task.result = json.dumps(ansible_result)
except json.JSONDecodeError:
publish_task.result = ansible_result
ss.flush()
pattern_task_id = publish_task.publish_pattern_task_id
if pattern_task_id != 0:
pattern_task = upgrade_pattern_task_status(
ss, pattern_task_id, ansible_status)
pattern_host_id = pattern_task.publish_pattern_host_id
pattern_hosts = upgrade_pattern_host_status(
ss, pattern_host_id)
pattern_id = pattern_hosts[0].publish_pattern_id
upgrade_pattern_status(ss, pattern_id)
if ansible_status == 'SUCCESS':
return True
def post(self):
argus = self.body_arguments
res = []
with session_scope() as ss:
for item in argus:
application_name = item.pop('application_name', None)
application_type = item.pop('application_type', None)
publish_host_ids = item.pop('publish_host_ids', None)
if not publish_host_ids or not application_name or not application_type:
raise HTTPError(status_code=400,
reason="Missing argument ,please check")
ready_host_tasks = []
for publish_host_id in publish_host_ids:
tasks = get_action_tasks(action=11,
application_type=application_type,
application_name=application_name)
tasks = [{
'publish_host_id_list': [publish_host_id],
'task_name': t
} for t in tasks]
ready_host_tasks.append(tasks)
host_and_id_list = run_chain_tasks(
session=ss, ready_host_tasks=ready_host_tasks)
res.extend(host_and_id_list)
for resource_id in publish_host_ids:
audit_log(self,
description='执行回滚',
resource_type=3,
resource_id=resource_id)
self.render_json_response(code=200, msg="OK", res=res)
def post(self):
'''
创建发版任务失败后，重试 下载与解压
:return:
'''
argument = self.body_arguments
publish_plan_id = argument.pop('publish_plan_id', None)
if publish_plan_id is None:
raise HTTPError(status_code=400, reason="Missing argument:publish_plan_id")
with session_scope() as ss:
plan = ss.query(PublishPlan).filter_by(id=publish_plan_id, status=3).one_or_none()
if plan is None:
raise HTTPError(status_code=400, reason="查找的publish_plan_id: {} 不存在，或者已创建成功 ".format(publish_plan_id))
# 状态重置为创建中
plan.status = 1
ss.flush()
inventory_version = plan.inventory_version
jenkins_url_list = [app.jenkins_url for app in plan.application_list]
task_id = uuid()
down_load_and_archive_package.apply_async(
kwargs={'jenkins_url_list': jenkins_url_list, 'inventory_version': inventory_version,
'publish_plan_id': publish_plan_id},
task_id=task_id)
audit_log(self, description='重试发版计划', resource_type=1, resource_id=publish_plan_id)
self.render_json_response(code=200, msg="OK", publish_plan_id=publish_plan_id, task_id=task_id)
def get_retry_rollback_pattern_ids(publish_plan_id):
"""
获取 未完成的 回滚步骤id
:param publish_plan_id:
:return:
"""
with session_scope() as ss:
patterns = ss.query(PublishPattern).filter(
PublishPattern.publish_plan_id == publish_plan_id,
PublishPattern.action.in_([11, 12, 14, 19]),
PublishPattern.status != 2).order_by(PublishPattern.step).all()
retry_pattern_id_list = []
for pattern in patterns:
# 有停顿，不再往下
if pattern.action == 19 and retry_pattern_id_list:
break
# status 任务状态 0: 待执行 1:执行中 2：完成 3：失败
if pattern.status == 1:
raise HTTPError(
status_code=400,
reason=
' {} with publish_plan_id {} and publish_pattern_id {} is running'
.format(pattern.title, publish_plan_id, pattern.id))
elif pattern.status == 2:
continue
else:
retry_pattern_id_list.append(pattern.id)
return retry_pattern_id_list
def callback(self):
if self.message is None:
return
publish_host_id = self.message['publish_host_id']
with session_scope() as ss:
q = ss.query(PublishTask).filter(
PublishTask.publish_host_id == publish_host_id)
db_res = q.limit(self.limit).offset(self.offset).all()
if db_res:
res = [r.to_dict() for r in db_res]
self.offset += len(db_res)
self.render_json_response(res=res)
else:
update_res = ss.query(PublishTask).filter(
PublishTask.publish_host_id == publish_host_id,
PublishTask.update_time > self.last_time).all()
if update_res:
res = [r.to_dict() for r in update_res]
self.last_time = datetime.datetime.now()
self.render_json_response(res=res)
def callback(self):
if self.message is None:
return
result = {'publish_task': [], 'publish_pattern_task': []}
publish_tasks = []
publish_pattern_host_id = self.message['publish_pattern_host_id']
with session_scope() as ss:
q = ss.query(PublishTask).join(
PublishPatternTask, PublishPatternTask.id ==
PublishTask.publish_pattern_task_id).filter(
PublishPatternTask.publish_pattern_host_id ==
publish_pattern_host_id)
db_res = q.limit(self.limit).offset(self.offset).all()
pattern_tasks = ss.query(PublishPatternTask).filter_by(
publish_pattern_host_id=publish_pattern_host_id).all()
publish_pattern_task = [t.to_dict() for t in pattern_tasks]
new_task_status = [t['status'] for t in publish_pattern_task]
if db_res:
for t in db_res:
if t.status in ('SUCCESS', 'FAILED'):
publish_tasks.append(t.to_dict())
result['publish_task'] = publish_tasks
self.offset += len(publish_tasks)
if publish_tasks or new_task_status != self.pattern_task_status:
self.pattern_task_status = new_task_status
result['publish_pattern_task'] = publish_pattern_task
self.render_json_response(res=result)
def get(self):
rightful_keys = (
'id',
'title',
'description',
'inventory_version',
'create_time',
'update_time',
'create_user',
'status',
'type',
'page',
'page_size',
)
uri_kwargs = take_out_unrightful_arguments(rightful_keys, self.url_arguments)
page = int(uri_kwargs.pop("page", 1))
page_size = int(uri_kwargs.pop("page_size", 5))
with session_scope() as ss:
publish_plan_query = ss.query(PublishPlan).filter_by(**uri_kwargs).filter(PublishPlan.status != 7).order_by(
desc(PublishPlan.create_time))
total_count = publish_plan_query.count()
publish_plan_list = publish_plan_query[(page - 1) * page_size:page * page_size]
res = [plan.to_dict() for plan in publish_plan_list]
self.render_json_response(code=200, msg="OK", total_count=total_count, res=res)
def test_create_project(self):
payload = {"name": "test_project"}
response = self.session.post(self.url, json=payload)
self.assertEqual(response.status_code, 200)
project_id = response.json()['res']['id']
with session_scope() as ss:
r = ss.query(PublishProject).filter_by(id=project_id).one_or_none()
self.assertNotEqual(r, None)
def test_delete_project(self):
payload = {"id": 1}
resp = self.session.delete(self.url, json=payload)
self.assertEqual(resp.status_code, 200, resp.json())
with session_scope() as ss:
db_res = ss.query(PublishProject).filter_by(id=payload['id']).one()
self.assertEqual(db_res.is_delete, 1, db_res)
def post(self):
"""
用户登录,获取token
"""
username = self.body_arguments.get('username')
password = self.body_arguments.get('password')
if not username or not password:
raise HTTPError(
status_code=400,
reason=
'Missing arguments,please check your username & password')
try:
user, error_reason = ldap.valid_user(username, password)
except LDAPException as e:
app_log.error(e)
raise HTTPError(status_code=500, reason='ldap error')
if not user:
raise HTTPError(status_code=400, reason=error_reason)
# update info
with session_scope() as ss:
# update user
user_instance = ss.query(User).filter(
User.username == user['username']).one_or_none()
if not user_instance:
user_instance = User()
ss.add(user_instance)
user_instance.username = user['username']
user_instance.fullname = user['fullname']
user_instance.email = user['email']
ss.flush()
# update department
department = user.pop('department_name', None)
# delete old user-department
ss.query(UserDepartment).filter(
UserDepartment.user_id == user_instance.id).delete()
if department is not None:
for dep_name in department:
department_instance = ss.query(Department).filter(
Department.name == dep_name).one_or_none()
if not department_instance:
department_instance = Department(name=dep_name)
ss.add(department_instance)
ss.flush()
ud = UserDepartment()
ud.department_id = department_instance.id
ud.user_id = user_instance.id
ss.add(ud)
ss.flush()
token = jwt.encode(payload=user_instance.to_dict(),
key=settings['token_secret'],
algorithm='HS256').decode("utf-8")
self.render_json_response(code=200, msg='OK', res={'token': token})
def on_success(self, retval, task_id, args, kwargs):
logger.info(
'create plan success and taskid is {} ,retval is{} ,args is{}.kwargs id {}'.format(task_id, retval, args,
kwargs))
with session_scope() as ss:
# 创建完成 更新publish plan 状态
publish_plan_id = kwargs['publish_plan_id']
ss.query(PublishPlan).filter(PublishPlan.id == publish_plan_id).update({'status': 2},
synchronize_session=False)
def on_failure(self, exc, task_id, args, kwargs, einfo):
err_msg = 'create plan {0!r} failed: {1!r}'.format(task_id, exc)
with session_scope() as ss:
# 任务失败 更新publish plan 状态
publish_plan_id = kwargs['publish_plan_id']
ss.query(PublishPlan).filter(PublishPlan.id == publish_plan_id).update({'status': 3},
synchronize_session=False)
# 任务失败时，把错误信息 打到log
logger.error(err_msg)
def get(self):
# take args
rightful_keys = ('id', 'create_time', 'update_time', 'user_id',
'resource_type', 'resource_id', 'description',
'visible', 'method', 'path', 'fullpath', 'body',
'page', 'page_size', 'order_by', 'desc',
'between_time', 'return_resource')
uri_kwargs = take_out_unrightful_arguments(rightful_keys,
self.url_arguments)
page = int(uri_kwargs.pop("page", 1))
page_size = int(uri_kwargs.pop("page_size", 15))
order_by = uri_kwargs.pop('order_by', None)
try:
desc_ = int(uri_kwargs.pop('desc', 0))
except ValueError:
desc_ = 0
try:
return_resource = int(uri_kwargs.pop('return_resource', 0))
except ValueError:
return_resource = 0
between_time = uri_kwargs.pop('between_time', None)
if between_time is not None:
uri_kwargs.pop('create_time', None)
between_time = between_time.split(',')
if len(between_time) < 2:
self.render_json_response(code=400,
msg='between_time need 2 arguments',
res=[])
with session_scope() as ss:
# join sql
q = ss.query(AuditLog).filter_by(**uri_kwargs)
if between_time:
q = q.filter(
AuditLog.create_time.between(between_time[0],
between_time[1]))
if order_by is not None:
if desc_:
order_by = desc(order_by)
q = q.order_by(order_by)
total_count = q.count()
res = pagination(q, page, page_size)
res = [r.to_dict(return_resource) for r in res]
return self.render_json_response(code=200,
msg='ok',
total_count=total_count,
res=res)
def delete(self):
_id = self.body_arguments.pop('id', None)
if _id is None:
raise HTTPError(status_code=400, reason='Missing argument "id"')
with session_scope() as ss:
ss.query(PublishProject).filter_by(id=_id).update({'is_delete': 1})
audit_log(self, description='删除发版项目', resource_type=5, resource_id=_id)
self.render_json_response(code=200, id=_id, res={'id': _id})
def get(self):
plan_id = self.url_arguments.get('planId')
with session_scope() as ss:
plan = ss.query(PublishPlan).get(plan_id)
if plan is not None:
res = plan.to_dict()
self.render_json_response(code=200, msg="OK", total_count=1, res=res)
else:
self.render_json_response(code=200, msg="OK", total_count=0, res={})
def check_pattern_status(pattern_id, **kwargs):
start = time.time()
while True and (time.time() - start) < 15 * 60:
with session_scope() as ss:
status = ss.query(
PublishPattern.status).filter_by(id=pattern_id).one()[0]
if status == 2: # 成功
return True
elif status == 3: # 失敗
return False
time.sleep(0.4)
return False
def get_host_by_publish_plan_id(publish_plan_id):
host_application_list_res = list()
host_application_dict = dict()
with session_scope() as ss:
query = ss.query(PublishHost, PublishApplication).join(
PublishApplication,
PublishApplication.id == PublishHost.publish_application_id).join(
PublishPatternHost,
PublishPatternHost.publish_host_id == PublishHost.id).join(
PublishPattern, PublishPattern.id ==
PublishPatternHost.publish_pattern_id).join(
PublishPlan,
PublishPlan.id == PublishPattern.publish_plan_id)
# PublishPattern.status 2: 完成 3：失败
# PublishPattern.action 1: 发布 2: 停服务 3: 提示
query = query.filter(PublishPattern.publish_plan_id == publish_plan_id,
PublishPlan.status.in_([4, 5, 8]),
PublishPattern.status.in_([2, 3]),
PublishPattern.action.in_([1, 2]))
db_res = query.all()
for ii in db_res:
host, application = ii
if host.rollback_version:
# 只有有rollback_version 的才可以进行回滚， 新应用无法进行回滚
_key = '{}_{}'.format(host.id, host.publish_application_id)
host_application_list = host_application_dict.setdefault(
_key, [])
host_application_list.append({
'publish_host_id':
host.id,
'host_ip':
host.host_ip,
'host_name':
host.host_name,
'rollback_version':
host.rollback_version,
'host_flag':
host.host_flag,
'application_name':
application.application_name,
'target_version':
application.target_version,
'application_type':
application.application_type,
})
for _list in list(host_application_dict.values()):
host_application_list_res.extend(_list)
return host_application_list_res
def get(self):
argus = self.url_arguments
taskids = argus.pop('taskids', None)
taskids = taskids.split(',')
if not taskids:
raise HTTPError(status_code=400,
reason='Missing arguments taskids')
with session_scope() as ss:
q = ss.query(PublishTask).filter(
PublishTask.celery_task_id.in_(taskids)).all()
res = [task.to_dict() for task in q]
self.render_json_response(code=200, msg='OK', res=res)
def callback(self):
if self.message is None or len(self.message['publish_plan_ids']) < 1:
return
with session_scope() as ss:
publish_plan_ids = self.message['publish_plan_ids']
publish_plans = ss.query(PublishPlan).filter(PublishPlan.id.in_(publish_plan_ids)).order_by(
PublishPlan.id).all()
new_status = [i.status for i in publish_plans]
if new_status != self.current_status:
res = [publish_plan.to_dict() for publish_plan in publish_plans]
self.render_json_response(res)
self.current_status = [i.status for i in publish_plans]
def delete(self):
""" delete user by id"""
arguments = self.body_arguments
if not arguments:
raise HTTPError(status_code=400,
reason="json arguments is invalid")
_id = arguments.pop("id", None)
if not _id:
raise HTTPError(status_code=400, reason="ID is required")
with session_scope() as ss:
ss.query(User).get(_id).delete()
self.render_json_response(code=200, msg="OK")
def delete(self):
""" delete publish plan by id"""
arguments = self.body_arguments
if not arguments:
raise HTTPError(status_code=400, reason="json arguments is invalid")
_id = arguments.pop("id", None)
if not _id:
raise HTTPError(status_code=400, reason="ID is required")
with session_scope() as ss:
ss.query(PublishPlan).filter_by(id=_id).update({'is_delete': 1})
audit_log(self, description='删除发版计划', resource_type=1, resource_id=_id)
self.render_json_response(code=200, msg='ok', res={'id': _id})
def put(self):
"""update user"""
arguments = self.body_arguments
if not arguments:
raise HTTPError(status_code=400,
reason="json arguments is invalid")
_id = arguments.pop("id", None)
if not _id:
raise HTTPError(status_code=400, reason="ID is required")
with session_scope() as ss:
ss.query(User).get(_id).update(arguments)
res = ss.query(User).get(_id).to_dict()
self.render_json_response(code=200, msg="OK", id=_id, res=res)
def callback(self):
if len(self.taskids) > 0:
try:
_temp_ids = []
with session_scope() as ss:
for task_id in self.taskids:
qq = ss.query(PublishTask).filter(
PublishTask.celery_task_id == task_id).first()
res = qq.to_dict() if qq else {}
if res.get('status') == 'SUCCESS':
self.render_json_response(res)
_temp_ids.append(task_id)
self.taskids = [x for x in self.taskids if x not in _temp_ids]
except Exception as inst:
self.render_json_response(
"Inter Server Error ： {}".format(inst))
