def post(self): argus = self.body_arguments publish_plan_id = argus.pop('publish_plan_id', None) if not publish_plan_id: raise HTTPError(status_code=400, reason='Missing arguments: publish_plan_id') res = [] with session_scope() as ss: publish_plan = ss.query(PublishPlan).filter_by( id=publish_plan_id).one_or_none() if not publish_plan or publish_plan.status != 21: raise HTTPError( status_code=400, reason= ' publish_plan_id {} is not exits or not in rollback status' .format(publish_plan_id)) retry_pattern_id_list = get_retry_rollback_pattern_ids(publish_plan_id) if not retry_pattern_id_list: raise HTTPError(status_code=400, reason="Has no retry_patterns to retry") ready_run_tasks = run_rollback_pattern(retry_pattern_id_list) with session_scope() as ss: db_res = ss.query(PublishPattern).filter( PublishPattern.id.in_(retry_pattern_id_list)).all() for pattern in db_res: r = pattern.to_dict() r['publish_application_hosts'] = get_publish_hosts(pattern) res.append(r) res.append({'ready_run_task': ready_run_tasks}) self.render_json_response(code=200, msg="OK", res=res)
def post(self): argus = self.body_arguments publish_plan_id = argus.pop('publish_plan_id', None) if not publish_plan_id: raise HTTPError(status_code=400, reason='Missing arguments: publish_plan_id') res = [] host_application_list = get_host_by_publish_plan_id(publish_plan_id) if not host_application_list: raise HTTPError( status_code=400, reason="publish_plan_id {} has no finished pattern".format( publish_plan_id)) all_pattern_id_list = generate_rollback_steps(publish_plan_id, host_application_list) with session_scope() as ss: # update publish plan status ss.query(PublishPlan).filter_by(id=publish_plan_id).update( {"status": 21}) # 把QA测试的步骤去掉 p_res = ss.query(PublishPattern).filter( PublishPattern.action > 10, PublishPattern.action != 13, PublishPattern.id.in_(all_pattern_id_list)).order_by( PublishPattern.step).all() pattern_id_list = [] for pattern in p_res: # 有停顿, 不再往下 if pattern.action == 19: break pattern_id_list.append(pattern.id) for pattern_id in pattern_id_list: audit_log(handler=self, description="执行回滚步骤", resource_type=4, resource_id=pattern_id) ready_run_tasks = run_rollback_pattern(pattern_id_list) with session_scope() as ss: db_res = ss.query(PublishPattern).filter( PublishPattern.id.in_(pattern_id_list)).all() for pattern in db_res: r = pattern.to_dict() r['publish_application_hosts'] = get_publish_hosts(pattern) res.append(r) res.append({'ready_run_task': ready_run_tasks}) self.render_json_response(code=200, msg="OK", res=res)
def get(self, *args, **kwargs): """ cmd: status/enable/disable """ argus = self.url_arguments pattern_id = argus.pop('pattern_id', None) cmdstr = argus.pop('cmd', 'status') project_name = argus.pop('application_name', None) publish_host_ids = argus.pop('publish_host_ids', '') if not publish_host_ids or not cmdstr: raise HTTPError( status=400, reason="Missing arguments publish_host_ids or cmdstr") publish_host_id_list = publish_host_ids.split(',') task_name = task_name_map[cmdstr] with session_scope() as ss: host_and_id_list = run_celery_task( session=ss, publish_host_id_list=publish_host_id_list, task_name=task_name, project_name=project_name, pattern_id=pattern_id) for resource_id in publish_host_id_list: audit_log(self, description=task_name, resource_type=3, resource_id=resource_id) self.render_json_response(code=200, msg="OK", res=host_and_id_list)
def put(self): res = {} rightful_keys = ( 'id', 'step', 'title', 'note', 'action', 'status', ) body_kwargs = take_out_unrightful_arguments(rightful_keys, self.body_arguments) id_ = body_kwargs.pop('id', None) if id_ is None: raise HTTPError(status_code=400, reason='Missing argument "id"') with session_scope() as ss: q = ss.query(PublishPattern).filter_by(id=id_) q.update(body_kwargs) db_res = q.one_or_none() if db_res is not None: res = db_res.to_dict() audit_log(self, description='更新发版步骤', resource_type=4, resource_id=id_) self.render_json_response(code=200, res=res)
def post(self, *args, **kwargs): argus = self.body_arguments if not isinstance(argus, list): raise HTTPError(status_code=400, reason='arguments must be list') res = [] with session_scope() as ss: for item in argus: pattern_id = item.pop('pattern_id', None) version = item['version'] project_name = item.pop('application_name', None) publish_host_ids = item.pop('publish_host_ids', None) if not (project_name or version or publish_host_ids or pattern_id): raise HTTPError(status_code=400, reason="Missing arguments") task_name = '执行发版' host_and_id_list = run_celery_task( session=ss, publish_host_id_list=publish_host_ids, task_name=task_name, pattern_id=pattern_id, version=version, project_name=project_name) res.extend(host_and_id_list) for resource_id in publish_host_ids: audit_log(self, description=task_name, resource_type=3, resource_id=resource_id) self.render_json_response(code=200, msg="OK", res=res)
def on_failure(self, exc, task_id, args, kwargs, einfo): logger.error('{0!r} failed: {1!r}'.format(task_id, exc)) argus = { "status": "FAILED", "result": str(exc), "end_time": datetime.now() } with session_scope() as ss: publish_task = ss.query(PublishTask).filter( PublishTask.celery_task_id == task_id).one_or_none() if publish_task is None: logger.error('PublishTask {} is not exist!'.format(task_id)) else: for k, v in argus.items(): publish_task.__setattr__(k, v) ss.flush() pattern_task_id = publish_task.publish_pattern_task_id if pattern_task_id != 0: pattern_task = upgrade_pattern_task_status( ss, pattern_task_id, argus['status']) pattern_host_id = pattern_task.publish_pattern_host_id pattern_hosts = upgrade_pattern_host_status( ss, pattern_host_id) pattern_id = pattern_hosts[0].publish_pattern_id upgrade_pattern_status(ss, pattern_id)
def do_success(self, retval, task_id, args, kwargs): argus = { "status": "SUCCESS", "result": json.dumps(retval) if retval else None, "end_time": datetime.now() } with session_scope() as ss: publish_task = ss.query(PublishTask).filter( PublishTask.celery_task_id == task_id).one_or_none() if publish_task is None: raise TaskError('PublishTask {} is not exist!'.format(task_id)) else: for k, v in argus.items(): publish_task.__setattr__(k, v) ss.flush() pattern_task_id = publish_task.publish_pattern_task_id if pattern_task_id != 0: pattern_task = upgrade_pattern_task_status( ss, pattern_task_id, argus['status']) pattern_host_id = pattern_task.publish_pattern_host_id pattern_hosts = upgrade_pattern_host_status( ss, pattern_host_id) pattern_id = pattern_hosts[0].publish_pattern_id upgrade_pattern_status(ss, pattern_id) return True
def do_success(self, retval, task_id, args, kwargs): with session_scope() as ss: publish_task = ss.query(PublishTask).filter( PublishTask.celery_task_id == task_id).one_or_none() if publish_task is None: raise TaskError('PublishTask {} is not exist!'.format(task_id)) else: ansible_result = self.get_ansible_result(retval) if ansible_result.get('rc') == 0: ansible_status = 'SUCCESS' else: ansible_status = 'FAILED' publish_task.status = ansible_status publish_task.end_time = datetime.now() try: publish_task.result = json.dumps(ansible_result) except json.JSONDecodeError: publish_task.result = ansible_result ss.flush() pattern_task_id = publish_task.publish_pattern_task_id if pattern_task_id != 0: pattern_task = upgrade_pattern_task_status( ss, pattern_task_id, ansible_status) pattern_host_id = pattern_task.publish_pattern_host_id pattern_hosts = upgrade_pattern_host_status( ss, pattern_host_id) pattern_id = pattern_hosts[0].publish_pattern_id upgrade_pattern_status(ss, pattern_id) if ansible_status == 'SUCCESS': return True
def post(self): argus = self.body_arguments res = [] with session_scope() as ss: for item in argus: application_name = item.pop('application_name', None) application_type = item.pop('application_type', None) publish_host_ids = item.pop('publish_host_ids', None) if not publish_host_ids or not application_name or not application_type: raise HTTPError(status_code=400, reason="Missing argument ,please check") ready_host_tasks = [] for publish_host_id in publish_host_ids: tasks = get_action_tasks(action=11, application_type=application_type, application_name=application_name) tasks = [{ 'publish_host_id_list': [publish_host_id], 'task_name': t } for t in tasks] ready_host_tasks.append(tasks) host_and_id_list = run_chain_tasks( session=ss, ready_host_tasks=ready_host_tasks) res.extend(host_and_id_list) for resource_id in publish_host_ids: audit_log(self, description='执行回滚', resource_type=3, resource_id=resource_id) self.render_json_response(code=200, msg="OK", res=res)
def post(self): ''' 创建发版任务失败后,重试 下载与解压 :return: ''' argument = self.body_arguments publish_plan_id = argument.pop('publish_plan_id', None) if publish_plan_id is None: raise HTTPError(status_code=400, reason="Missing argument:publish_plan_id") with session_scope() as ss: plan = ss.query(PublishPlan).filter_by(id=publish_plan_id, status=3).one_or_none() if plan is None: raise HTTPError(status_code=400, reason="查找的publish_plan_id: {} 不存在,或者已创建成功 ".format(publish_plan_id)) # 状态重置为创建中 plan.status = 1 ss.flush() inventory_version = plan.inventory_version jenkins_url_list = [app.jenkins_url for app in plan.application_list] task_id = uuid() down_load_and_archive_package.apply_async( kwargs={'jenkins_url_list': jenkins_url_list, 'inventory_version': inventory_version, 'publish_plan_id': publish_plan_id}, task_id=task_id) audit_log(self, description='重试发版计划', resource_type=1, resource_id=publish_plan_id) self.render_json_response(code=200, msg="OK", publish_plan_id=publish_plan_id, task_id=task_id)
def get_retry_rollback_pattern_ids(publish_plan_id): """ 获取 未完成的 回滚步骤id :param publish_plan_id: :return: """ with session_scope() as ss: patterns = ss.query(PublishPattern).filter( PublishPattern.publish_plan_id == publish_plan_id, PublishPattern.action.in_([11, 12, 14, 19]), PublishPattern.status != 2).order_by(PublishPattern.step).all() retry_pattern_id_list = [] for pattern in patterns: # 有停顿,不再往下 if pattern.action == 19 and retry_pattern_id_list: break # status 任务状态 0: 待执行 1:执行中 2:完成 3:失败 if pattern.status == 1: raise HTTPError( status_code=400, reason= ' {} with publish_plan_id {} and publish_pattern_id {} is running' .format(pattern.title, publish_plan_id, pattern.id)) elif pattern.status == 2: continue else: retry_pattern_id_list.append(pattern.id) return retry_pattern_id_list
def callback(self): if self.message is None: return publish_host_id = self.message['publish_host_id'] with session_scope() as ss: q = ss.query(PublishTask).filter( PublishTask.publish_host_id == publish_host_id) db_res = q.limit(self.limit).offset(self.offset).all() if db_res: res = [r.to_dict() for r in db_res] self.offset += len(db_res) self.render_json_response(res=res) else: update_res = ss.query(PublishTask).filter( PublishTask.publish_host_id == publish_host_id, PublishTask.update_time > self.last_time).all() if update_res: res = [r.to_dict() for r in update_res] self.last_time = datetime.datetime.now() self.render_json_response(res=res)
def callback(self): if self.message is None: return result = {'publish_task': [], 'publish_pattern_task': []} publish_tasks = [] publish_pattern_host_id = self.message['publish_pattern_host_id'] with session_scope() as ss: q = ss.query(PublishTask).join( PublishPatternTask, PublishPatternTask.id == PublishTask.publish_pattern_task_id).filter( PublishPatternTask.publish_pattern_host_id == publish_pattern_host_id) db_res = q.limit(self.limit).offset(self.offset).all() pattern_tasks = ss.query(PublishPatternTask).filter_by( publish_pattern_host_id=publish_pattern_host_id).all() publish_pattern_task = [t.to_dict() for t in pattern_tasks] new_task_status = [t['status'] for t in publish_pattern_task] if db_res: for t in db_res: if t.status in ('SUCCESS', 'FAILED'): publish_tasks.append(t.to_dict()) result['publish_task'] = publish_tasks self.offset += len(publish_tasks) if publish_tasks or new_task_status != self.pattern_task_status: self.pattern_task_status = new_task_status result['publish_pattern_task'] = publish_pattern_task self.render_json_response(res=result)
def get(self): rightful_keys = ( 'id', 'title', 'description', 'inventory_version', 'create_time', 'update_time', 'create_user', 'status', 'type', 'page', 'page_size', ) uri_kwargs = take_out_unrightful_arguments(rightful_keys, self.url_arguments) page = int(uri_kwargs.pop("page", 1)) page_size = int(uri_kwargs.pop("page_size", 5)) with session_scope() as ss: publish_plan_query = ss.query(PublishPlan).filter_by(**uri_kwargs).filter(PublishPlan.status != 7).order_by( desc(PublishPlan.create_time)) total_count = publish_plan_query.count() publish_plan_list = publish_plan_query[(page - 1) * page_size:page * page_size] res = [plan.to_dict() for plan in publish_plan_list] self.render_json_response(code=200, msg="OK", total_count=total_count, res=res)
def test_create_project(self): payload = {"name": "test_project"} response = self.session.post(self.url, json=payload) self.assertEqual(response.status_code, 200) project_id = response.json()['res']['id'] with session_scope() as ss: r = ss.query(PublishProject).filter_by(id=project_id).one_or_none() self.assertNotEqual(r, None)
def test_delete_project(self): payload = {"id": 1} resp = self.session.delete(self.url, json=payload) self.assertEqual(resp.status_code, 200, resp.json()) with session_scope() as ss: db_res = ss.query(PublishProject).filter_by(id=payload['id']).one() self.assertEqual(db_res.is_delete, 1, db_res)
def post(self): """ 用户登录,获取token """ username = self.body_arguments.get('username') password = self.body_arguments.get('password') if not username or not password: raise HTTPError( status_code=400, reason= 'Missing arguments,please check your username & password') try: user, error_reason = ldap.valid_user(username, password) except LDAPException as e: app_log.error(e) raise HTTPError(status_code=500, reason='ldap error') if not user: raise HTTPError(status_code=400, reason=error_reason) # update info with session_scope() as ss: # update user user_instance = ss.query(User).filter( User.username == user['username']).one_or_none() if not user_instance: user_instance = User() ss.add(user_instance) user_instance.username = user['username'] user_instance.fullname = user['fullname'] user_instance.email = user['email'] ss.flush() # update department department = user.pop('department_name', None) # delete old user-department ss.query(UserDepartment).filter( UserDepartment.user_id == user_instance.id).delete() if department is not None: for dep_name in department: department_instance = ss.query(Department).filter( Department.name == dep_name).one_or_none() if not department_instance: department_instance = Department(name=dep_name) ss.add(department_instance) ss.flush() ud = UserDepartment() ud.department_id = department_instance.id ud.user_id = user_instance.id ss.add(ud) ss.flush() token = jwt.encode(payload=user_instance.to_dict(), key=settings['token_secret'], algorithm='HS256').decode("utf-8") self.render_json_response(code=200, msg='OK', res={'token': token})
def on_success(self, retval, task_id, args, kwargs): logger.info( 'create plan success and taskid is {} ,retval is{} ,args is{}.kwargs id {}'.format(task_id, retval, args, kwargs)) with session_scope() as ss: # 创建完成 更新publish plan 状态 publish_plan_id = kwargs['publish_plan_id'] ss.query(PublishPlan).filter(PublishPlan.id == publish_plan_id).update({'status': 2}, synchronize_session=False)
def on_failure(self, exc, task_id, args, kwargs, einfo): err_msg = 'create plan {0!r} failed: {1!r}'.format(task_id, exc) with session_scope() as ss: # 任务失败 更新publish plan 状态 publish_plan_id = kwargs['publish_plan_id'] ss.query(PublishPlan).filter(PublishPlan.id == publish_plan_id).update({'status': 3}, synchronize_session=False) # 任务失败时,把错误信息 打到log logger.error(err_msg)
def get(self): # take args rightful_keys = ('id', 'create_time', 'update_time', 'user_id', 'resource_type', 'resource_id', 'description', 'visible', 'method', 'path', 'fullpath', 'body', 'page', 'page_size', 'order_by', 'desc', 'between_time', 'return_resource') uri_kwargs = take_out_unrightful_arguments(rightful_keys, self.url_arguments) page = int(uri_kwargs.pop("page", 1)) page_size = int(uri_kwargs.pop("page_size", 15)) order_by = uri_kwargs.pop('order_by', None) try: desc_ = int(uri_kwargs.pop('desc', 0)) except ValueError: desc_ = 0 try: return_resource = int(uri_kwargs.pop('return_resource', 0)) except ValueError: return_resource = 0 between_time = uri_kwargs.pop('between_time', None) if between_time is not None: uri_kwargs.pop('create_time', None) between_time = between_time.split(',') if len(between_time) < 2: self.render_json_response(code=400, msg='between_time need 2 arguments', res=[]) with session_scope() as ss: # join sql q = ss.query(AuditLog).filter_by(**uri_kwargs) if between_time: q = q.filter( AuditLog.create_time.between(between_time[0], between_time[1])) if order_by is not None: if desc_: order_by = desc(order_by) q = q.order_by(order_by) total_count = q.count() res = pagination(q, page, page_size) res = [r.to_dict(return_resource) for r in res] return self.render_json_response(code=200, msg='ok', total_count=total_count, res=res)
def delete(self): _id = self.body_arguments.pop('id', None) if _id is None: raise HTTPError(status_code=400, reason='Missing argument "id"') with session_scope() as ss: ss.query(PublishProject).filter_by(id=_id).update({'is_delete': 1}) audit_log(self, description='删除发版项目', resource_type=5, resource_id=_id) self.render_json_response(code=200, id=_id, res={'id': _id})
def get(self): plan_id = self.url_arguments.get('planId') with session_scope() as ss: plan = ss.query(PublishPlan).get(plan_id) if plan is not None: res = plan.to_dict() self.render_json_response(code=200, msg="OK", total_count=1, res=res) else: self.render_json_response(code=200, msg="OK", total_count=0, res={})
def check_pattern_status(pattern_id, **kwargs): start = time.time() while True and (time.time() - start) < 15 * 60: with session_scope() as ss: status = ss.query( PublishPattern.status).filter_by(id=pattern_id).one()[0] if status == 2: # 成功 return True elif status == 3: # 失敗 return False time.sleep(0.4) return False
def get_host_by_publish_plan_id(publish_plan_id): host_application_list_res = list() host_application_dict = dict() with session_scope() as ss: query = ss.query(PublishHost, PublishApplication).join( PublishApplication, PublishApplication.id == PublishHost.publish_application_id).join( PublishPatternHost, PublishPatternHost.publish_host_id == PublishHost.id).join( PublishPattern, PublishPattern.id == PublishPatternHost.publish_pattern_id).join( PublishPlan, PublishPlan.id == PublishPattern.publish_plan_id) # PublishPattern.status 2: 完成 3:失败 # PublishPattern.action 1: 发布 2: 停服务 3: 提示 query = query.filter(PublishPattern.publish_plan_id == publish_plan_id, PublishPlan.status.in_([4, 5, 8]), PublishPattern.status.in_([2, 3]), PublishPattern.action.in_([1, 2])) db_res = query.all() for ii in db_res: host, application = ii if host.rollback_version: # 只有有rollback_version 的才可以进行回滚, 新应用无法进行回滚 _key = '{}_{}'.format(host.id, host.publish_application_id) host_application_list = host_application_dict.setdefault( _key, []) host_application_list.append({ 'publish_host_id': host.id, 'host_ip': host.host_ip, 'host_name': host.host_name, 'rollback_version': host.rollback_version, 'host_flag': host.host_flag, 'application_name': application.application_name, 'target_version': application.target_version, 'application_type': application.application_type, }) for _list in list(host_application_dict.values()): host_application_list_res.extend(_list) return host_application_list_res
def get(self): argus = self.url_arguments taskids = argus.pop('taskids', None) taskids = taskids.split(',') if not taskids: raise HTTPError(status_code=400, reason='Missing arguments taskids') with session_scope() as ss: q = ss.query(PublishTask).filter( PublishTask.celery_task_id.in_(taskids)).all() res = [task.to_dict() for task in q] self.render_json_response(code=200, msg='OK', res=res)
def callback(self): if self.message is None or len(self.message['publish_plan_ids']) < 1: return with session_scope() as ss: publish_plan_ids = self.message['publish_plan_ids'] publish_plans = ss.query(PublishPlan).filter(PublishPlan.id.in_(publish_plan_ids)).order_by( PublishPlan.id).all() new_status = [i.status for i in publish_plans] if new_status != self.current_status: res = [publish_plan.to_dict() for publish_plan in publish_plans] self.render_json_response(res) self.current_status = [i.status for i in publish_plans]
def delete(self): """ delete user by id""" arguments = self.body_arguments if not arguments: raise HTTPError(status_code=400, reason="json arguments is invalid") _id = arguments.pop("id", None) if not _id: raise HTTPError(status_code=400, reason="ID is required") with session_scope() as ss: ss.query(User).get(_id).delete() self.render_json_response(code=200, msg="OK")
def delete(self): """ delete publish plan by id""" arguments = self.body_arguments if not arguments: raise HTTPError(status_code=400, reason="json arguments is invalid") _id = arguments.pop("id", None) if not _id: raise HTTPError(status_code=400, reason="ID is required") with session_scope() as ss: ss.query(PublishPlan).filter_by(id=_id).update({'is_delete': 1}) audit_log(self, description='删除发版计划', resource_type=1, resource_id=_id) self.render_json_response(code=200, msg='ok', res={'id': _id})
def put(self): """update user""" arguments = self.body_arguments if not arguments: raise HTTPError(status_code=400, reason="json arguments is invalid") _id = arguments.pop("id", None) if not _id: raise HTTPError(status_code=400, reason="ID is required") with session_scope() as ss: ss.query(User).get(_id).update(arguments) res = ss.query(User).get(_id).to_dict() self.render_json_response(code=200, msg="OK", id=_id, res=res)
def callback(self): if len(self.taskids) > 0: try: _temp_ids = [] with session_scope() as ss: for task_id in self.taskids: qq = ss.query(PublishTask).filter( PublishTask.celery_task_id == task_id).first() res = qq.to_dict() if qq else {} if res.get('status') == 'SUCCESS': self.render_json_response(res) _temp_ids.append(task_id) self.taskids = [x for x in self.taskids if x not in _temp_ids] except Exception as inst: self.render_json_response( "Inter Server Error : {}".format(inst))