def __new__(cls, name, bases, attrs): def _make_method(action): def method(self, context, x, y): context.finish() return Result.ok(data=(x + y)) return method attrs["on_trigger"] = _make_method("trigger") attrs["__init__"] = lambda self: AbstractJob.__init__( self, "job_" + letter, "job " + letter, job_args={ "trigger": [ JobArg("x", IntField("x", required=True), JobArg. MARK_AUTO, "arg x"), JobArg("y", IntField("y", required=True), JobArg. MARK_AUTO, "arg y") ] }) bases += (AbstractJob, ) return type(name, bases, attrs)
def testIntField(self): """测试IntField类型的规则 """ int_field = IntField("age", min_=6, max_=100) # 值正常的时候 self.assertEqual(int_field(99), 99) self.assertEqual(int_field("10"), 10) # 类型不匹配 with self.assertRaises(InvalidFieldException) as raise_ctx: int_field("nidaye") self._assert_invalid_f( raise_ctx.exception, "age", "nidaye", "invalid_type", int) # 大小不匹配 with self.assertRaises(InvalidFieldException) as raise_ctx: int_field(-1) self._assert_invalid_f( raise_ctx.exception, "age", -1, "less_than_min", 6) with self.assertRaises(InvalidFieldException) as raise_ctx: int_field(101) self._assert_invalid_f( raise_ctx.exception, "age", 101, "more_than_max", 100) # 自定义检查逻辑 def _my_check(field_name, age): if age % 2 != 0: raise InvalidFieldException( field_name, age, "invalid_age", "even") int_field = IntField("age", min_=6, max_=100, check_logic=_my_check) # 值正确的情况 self.assertEqual(int_field(6), 6) self.assertEqual(int_field(100), 100) # 值不正确的情况 with self.assertRaises(InvalidFieldException) as raise_ctx: int_field(7) self._assert_invalid_f( raise_ctx.exception, "age", 7, "invalid_age", "even") # required but value is None with self.assertRaises(InvalidFieldException) as raise_ctx: int_field(None) self._assert_invalid_f(raise_ctx.exception, "age", None, "empty", "") # not required and value is None int_rule = IntField("age", False, default=6) self.assertEqual(int_rule(None), 6)
class TestFlowMeta(FlowMeta): def __init__(self): super().__init__( name="test_flow", description="just a test flow", jobs=( JobRef( step_name="echo", job_name="echo", trigger={ "a": 5 }, finish={ }, stop={ } ), JobRef( step_name="old_man", job_name="old_man" ), JobRef( step_name="job_A", job_name="job_A" ), ), start_args={ "x": -1, "y": -2 }, ) @declare_args( IntField("x", required=True), IntField("y", required=True) ) def on_start(self, context, x, y): print("start the workflow, x = {x}, y = {y}".format( x=x, y=y )) def on_stop(self, context): print("the whole workflow stopped") def on_finish(self, context): print("the whole workflow finished")
class BService: @check( IntField("id"), messages=self.messages ) def b(self, id): if id < 1: raise BadReq("invalid_id", id=id)
class AService: @check( IntField("id", required=True, min_=6, max_=100), StrField("name", required=True, min_len=3, max_len=20), StrField("grade", required=False, default="X", regex=r'^[A-Z]$'), messages=self.messages ) def a(self, id: int, name: str, grade: str) -> typing.Tuple: return id, name, grade
def __init__(self): super().__init__("echo", "test job echo", job_args={ "trigger": [ JobArg("a", IntField("a", required=True), JobArg.MARK_CONST, "a的值"), JobArg("b", IntField("b", required=True), JobArg.MARK_STATIC, "b的值"), ], "multiply": [ JobArg("c", IntField("c", required=True), JobArg.MARK_AUTO, "c的值") ], "minus": [ JobArg("d", IntField("d", required=True), JobArg.MARK_AUTO, "d的值"), JobArg("e", IntField("e", required=True), JobArg.MARK_AUTO, "e的值") ], })
def __init__(self): super().__init__(name="programmer", description=("我是一个程序员," "我的爱好是抢月饼!"), job_args={ "midautumn": [ JobArg( "cake_num", IntField("cake_num", required=False, default=1, min_=1), mark=JobArg.MARK_AUTO, comment="抢到的月饼数目", ), ] })
class FlowExecutorService(AbstractService): def __init__(self, service_container: ServiceContainer): super().__init__(service_container) def _after_register(self): # 获取各种所依赖的服务 self._db = self._("db") self._flow_tpl_dao = FlowTemplateDAO(self._db) self._flow_meta_mgr = self._("flow_meta_manager") self._job_mgr = self._("job_manager") self._flow_instance_dao = FlowInstanceDAO(self._db) self._user_dao = UserDAO(self._db) self._job_instance_dao = JobInstanceDAO(self._db) self._job_action_dao = JobActionDataDAO(self._db) @check( IntField("flow_template_id", required=True), IntField("initiator", required=True), StrField("description", required=True, max_len=20), Field("start_flow_args", type_=dict, required=False, default=None, value_of=json.loads), ) def start_flow(self, flow_template_id: int, initiator: int, description: str, start_flow_args: Dict[str, Any]) -> Result: """开启一个flow进程,创建flow_instance并执行第一个Job S1. 根据flow_template_id获取flow_template,然后获取flow_meta,如果获取失败,返回错误 S2. 检查并合并参数 S3. 检查max_run_instance S4. 创建一条新的flow_instance记录 S5. 创建context S6. 回调flow meta中on_start方法的逻辑 :param flow_template_id: 使用的flow template :param initiator: 发起人 :param description: 本次flow描述 :param start_flow_args: 执行FlowMeta的on_start方法时所需要的参数 """ if start_flow_args is None: start_flow_args = {} # 检查flow_template_id是否合法 flow_template = self._flow_tpl_dao.query_flow_template_by_id( flow_template_id) if flow_template is None: raise BadReq("invalid_flow_template", flow_template_id=flow_template_id) flow_meta = self._flow_meta_mgr.get(flow_template.flow_meta) if flow_meta is None: raise BadReq("invalid_flow_meta", flow_meta=flow_meta) # 检查发起者 initiator_user = self._user_dao.query_user_by_id(initiator) if initiator_user is None: raise BadReq("invalid_initiator", initiator=initiator) # 检查并合并start_flow_args field_rules = getattr(flow_meta.on_start, "field_rules", []) rs = self._combine_and_check_args("start", field_rules, start_flow_args, flow_meta.start_args) if rs.status_code == Result.STATUS_BADREQUEST: return rs start_flow_args = rs.data # 锁定检查instance数目并创建第一条记录 flow_instance = None if flow_template.max_run_instance > 0: lock_key = "lock_instance_create_{tpl_id}".format( tpl_id=flow_template_id) with self._db.lock(lock_key): current_instance_num = self._flow_instance_dao.\ query_running_instance_num_by_tpl_id(flow_template_id) if current_instance_num >= flow_template.max_run_instance: raise BadReq( reason="too_many_instance", allow_instance_num=flow_template.max_run_instance) flow_instance = self._flow_instance_dao.insert( flow_template_id, initiator, description) else: flow_instance = self._flow_instance_dao.insert( flow_template_id, initiator, description) # 创建Context ctx = MySQLContext(self, self._db, flow_instance.id) # 回调on_start flow_meta.on_start(ctx, **start_flow_args) # 将状态更新到running self._flow_instance_dao.update_status(flow_instance.id, FlowStatus.STATUS_RUNNING) log.acolyte.info("start flow instance {}".format( to_json(flow_instance))) return Result.ok(data=flow_instance) @check(IntField("flow_instance_id", required=True), StrField("target_step", required=True), StrField("target_action", required=True), IntField("actor", required=True), Field("action_args", type_=dict, required=False, default=None, value_of=json.loads)) def handle_job_action(self, flow_instance_id: int, target_step: str, target_action: str, actor: int, action_args: Dict[str, Any]) -> Result: """处理Job中的Action S1. 检查并获取flow实例 S2. 检查job以及job_action的存在性 S3. 检查执行人是否合法 S4. 检查当前是否可以允许该step及target_action的执行 S5. 合并以及检查相关参数 S6. 回调相关Action逻辑 S7. 返回回调函数的返回值 :param flow_instance_id: flow的标识 :param target_step: 要执行的Step :param target_action: 自定义的动作名称 :param actor: 执行人 :param action_args: 执行该自定义动作所需要的参数 """ if action_args is None: action_args = {} # 检查flow instance的id合法性 flow_instance = self._flow_instance_dao.query_by_instance_id( flow_instance_id) if flow_instance is None: raise BadReq("invalid_flow_instance", flow_instance_id=flow_instance_id) # 检查flow instance的状态 if flow_instance.status != FlowStatus.STATUS_RUNNING: raise BadReq("invalid_status", status=flow_instance.status) # 获取对应的flow template和flow meta flow_template = self._flow_tpl_dao\ .query_flow_template_by_id(flow_instance.flow_template_id) if flow_template is None: raise BadReq("unknown_flow_template", flow_template_id=flow_instance.flow_template_id) try: flow_meta = self._flow_meta_mgr.get(flow_template.flow_meta) except ObjectNotFoundException: raise BadReq("unknown_flow_meta", flow_meta=flow_meta) actor_info = self._user_dao.query_user_by_id(actor) if actor_info is None: raise BadReq("invalid_actor", actor=actor) # 检查当前step以及当前step是否完成 # 检查下一个状态是否是目标状态 handler_mtd, job_def, job_ref = self._check_step( flow_meta, flow_instance, target_step, target_action) # 合并检查参数 request_args - template_bind_args - meta_bind_args rs = self._check_and_combine_action_args(job_def, target_action, action_args, job_ref, flow_template) if rs.status_code == Result.STATUS_BADREQUEST: return rs job_instance = self._job_instance_dao.query_by_instance_id_and_step( instance_id=flow_instance_id, step=target_step) # 如果是trigger事件,需要创建job_instance记录 if target_action == "trigger": job_instance = self._job_instance_dao.insert( flow_instance_id, target_step, actor) self._flow_instance_dao.update_current_step( flow_instance_id, target_step) action_args = rs.data action = self._job_action_dao.insert(job_instance_id=job_instance.id, action=target_action, actor=actor, arguments=action_args, data={}) ctx = MySQLContext(flow_executor=self, db=self._db, flow_instance_id=flow_instance.id, job_instance_id=job_instance.id, job_action_id=action.id, flow_meta=flow_meta, current_step=target_step) rs = handler_mtd(ctx, **action_args) if not isinstance(rs, Result): rs = Result.ok(data=rs) if not rs.is_success(): # 如果返回结果不成功,那么允许重来 self._job_action_dao.delete_by_id(action.id) log.acolyte.info(("Job action executed, " "action_data = {action_data}, " "action_result = {action_result}").format( action_data=to_json(action), action_result=to_json(rs))) return rs def _check_and_combine_action_args(self, job_def, target_action, request_args, job_ref, flow_template): job_arg_defines = job_def.job_args.get(target_action) # 无参数定义 if not job_arg_defines: return Result.ok(data={}) # 获取各级的参数绑定 meta_bind_args = job_ref.bind_args.get(target_action, {}) tpl_bind_args = flow_template.bind_args.get(job_ref.step_name, {}).get(target_action, {}) args_chain = ChainMap(request_args, tpl_bind_args, meta_bind_args) # 最终生成使用的参数集合 args = {} for job_arg_define in job_arg_defines: try: arg_name = job_arg_define.name # auto 类型,直接从chain中取值 if job_arg_define.mark == JobArg.MARK_AUTO: value = args_chain[arg_name] # static类型,从template中取值 elif job_arg_define.mark == JobArg.MARK_STATIC: value = tpl_bind_args.get(arg_name, None) # const类型,从meta中取值 elif job_arg_define.mark == JobArg.MARK_CONST: value = meta_bind_args.get(arg_name, None) args[arg_name] = job_arg_define.field_info(value) except InvalidFieldException as e: full_field_name = "{step}.{action}.{arg}".format( step=job_ref.step_name, action=target_action, arg=arg_name) return self._gen_bad_req_result(e, full_field_name) return Result.ok(data=args) def _check_step(self, flow_meta, flow_instance, target_step, target_action): current_step = flow_instance.current_step # 检查当前action的方法是否存在 target_job_ref = flow_meta.get_job_ref_by_step_name(target_step) if target_job_ref is None: raise BadReq("unknown_target_step", target_step=target_step) try: job_def = self._job_mgr.get(target_job_ref.job_name) except ObjectNotFoundException: raise BadReq("unknown_job", job_name=target_job_ref.name) handler_mtd = getattr(job_def, "on_" + target_action, None) if handler_mtd is None: raise BadReq("unknown_action_handler", action=target_action) # 当前step即目标step if current_step == target_step: job_instance = self._job_instance_dao.\ query_by_instance_id_and_step( instance_id=flow_instance.id, step=current_step ) if job_instance.status == JobStatus.STATUS_FINISHED: raise BadReq("step_already_runned", step=target_step) # 检查当前action是否被执行过 action = self._job_action_dao.\ query_by_job_instance_id_and_action( job_instance_id=job_instance.id, action=target_action ) if action is not None: raise BadReq("action_already_runned", action=target_action) # 如果非trigger,则检查trigger是否执行过 if target_action != "trigger": trigger_action = self._job_action_dao.\ query_by_job_instance_id_and_action( job_instance_id=job_instance.id, action="trigger" ) if trigger_action is None: raise BadReq("no_trigger") return handler_mtd, job_def, target_job_ref if current_step != "start": # 当前step非目标step job_instance = self._job_instance_dao.\ query_by_instance_id_and_step( instance_id=flow_instance.id, step=current_step ) # 流程记录了未知的current_step if job_instance is None: raise BadReq("unknown_current_step", current_step=current_step) # 当前的step尚未完成 if job_instance.status != JobStatus.STATUS_FINISHED: raise BadReq("current_step_unfinished", current_step=current_step) # 获取下一个该运行的步骤 next_step = flow_meta.get_next_step(current_step) if next_step != target_step: raise BadReq("invalid_target_step", next_step=next_step) if target_action != "trigger": raise BadReq("no_trigger") return handler_mtd, job_def, target_job_ref def _combine_and_check_args(self, action_name, field_rules, *args_dict): """合并 & 检查参数 先合并,后检查 :param field_rules: 字段规则 :param old_args """ _combined_args = ChainMap(*args_dict).new_child() # 木有验证规则,直接返回数据 if not field_rules: return Result.ok(data=_combined_args) try: for field_rule in field_rules: # 检查并替换掉相关参数 val = field_rule(_combined_args[field_rule.name]) _combined_args[field_rule.name] = val except InvalidFieldException as e: full_field_name = "{action_name}.{field_name}".format( action_name=action_name, field_name=e.field_name) return self._gen_bad_req_result(e, full_field_name) else: return Result.ok(data=_combined_args) def _gen_bad_req_result(self, e, full_field_name): loc, _ = locale.getlocale(locale.LC_ALL) full_reason = "{full_field_name}_{reason}".format( full_field_name=full_field_name, reason=e.reason) msg = default_validate_messages[loc][e.reason] if e.expect is None or e.expect == "": msg = msg.format(field_name=full_field_name) else: msg = msg.format(field_name=full_field_name, expect=e.expect) return Result.bad_request(reason=full_reason, msg=msg) def _finish_step(self, ctx): """标记一个job instance完成,通常由action通过context进行回调 S1. 将job_instance的状态更新为finish S2. 检查整个flow是否已经完成 S3. 如果整个流程已经完成,那么标记flow_instance的status S4. 回调flow_meta中的on_finish事件 """ flow_instance_id, job_instance_id = (ctx.flow_instance_id, ctx.job_instance_id) self._job_instance_dao.update_status(job_instance_id=job_instance_id, status=JobStatus.STATUS_FINISHED) next_step = ctx.flow_meta.get_next_step(ctx.current_step) # 尚未完成,继续处理 if next_step != "finish": return # 修改flow_instance的状态 self._flow_instance_dao.update_status( flow_instance_id=flow_instance_id, status=FlowStatus.STATUS_FINISHED) # 回调on_finish事件 on_finish_handler = getattr(ctx.flow_meta, "on_finish", None) if on_finish_handler is None: return on_finish_handler(ctx) def _stop_whole_flow(self, ctx): """终止整个flow的运行,通常由action通过context进行回调 S1. 标记flow_instance的status为stop S2. 回调flow_meta中的on_stop事件 """ self._flow_instance_dao.update_status( flow_instance_id=ctx.flow_instance_id, status=FlowStatus.STATUS_STOPPED) # 回调on_stop事件 on_stop_handler = getattr(ctx.flow_meta, "on_stop", None) if on_stop_handler is None: return on_stop_handler(ctx)
class UserService(AbstractService): _TOKEN_SALT = "6f81900c31f7fd80bd" def __init__(self, service_container): super().__init__(service_container) def _after_register(self): self._db = self._("db") self._user_dao = UserDAO(self._db) self._role_dao = RoleDAO(self._db) self._user_token_dao = UserTokenDAO(self._db) @check( StrField("email", required=True), StrField("password", required=True) ) def login(self, email: str, password: str) -> Result: """登录 S1. 通过email和password检索用户 S2. 创建并获取新的token S3. 存储相关用户数据到session_data """ user = self._user_dao.query_user_by_email_and_password( email=email, password=sha1(password) ) if user is None: raise BadReq("no_match") # do upsert new_token = self._gen_new_token(user.id) self._user_token_dao.upsert_token(user.id, new_token) # save user basic info to session data self._user_token_dao.save_session_data( new_token, name=user.name, email=user.email) return Result.ok(data={"id": user.id, "token": new_token}) def _gen_new_token(self, user_id: int): """生成新token 规则: sha1({用户ID}{时间戳}{随机数}{salt}) """ return sha1(( "{user_id}" "{timestamp_int}" "{randnum}" "{salt}" ).format( user_id=user_id, timestamp_int=int(time.time()), randnum=random.randint(10000, 99999), salt=UserService._TOKEN_SALT)) @check( StrField("email", required=True, regex=r'^[\w.-]+@[\w.-]+.\w+$'), StrField("password", required=True, min_len=6, max_len=20), StrField("name", required=True, max_len=10), IntField("role", required=True), IntField("operator", required=True) ) def add_user(self, email: str, password: str, name: str, role: int, operator: int) -> Result: """添加新用户 S1. 检查邮箱是否存在 S2. 检查角色是否存在 S3. 检查operator是否有权限 S4. 创建新用户 :param email: 邮箱地址 :param password: 密码,会经过sha1加密 :param name: 姓名 :param role: 角色编号 :param operator: 操作者 """ # 检查是否已注册 if self._user_dao.is_email_exist(email): raise BadReq("email_exist", email=email) # 检查角色是否合法 if not self._role_dao.query_role_by_id(role): raise BadReq("role_not_found", role=role) # 检查操作者信息及权限 operator_model = self._user_dao.query_user_by_id(operator) if operator_model is None: raise BadReq("operator_not_found") operator_role = self._role_dao.query_role_by_id(operator_model.role) if operator_role.name != "admin": raise BadReq("not_allow_operation") # 创建新用户 new_user = self._user_dao.insert_user( email, sha1(password), name, role) return Result.ok(data=new_user) @check(StrField("token", required=True)) def check_token(self, token: str) -> Result: """检查token S1. 查找token相关的用户信息 S2. 返回token关联简单会话数据 """ session_data = self._user_token_dao.query_session_data(token) if session_data is None: raise BadReq("invalid_token") return Result.ok(data=session_data) def logout(self, token: str) -> Result: """退出 S1. 直接从数据库中删除token记录 """ self._user_token_dao.delete_by_token(token) return Result.ok() def profile(self, user_id: int) -> Result: ...
class FlowService(AbstractService): def __init__(self, service_container): super().__init__(service_container) def _after_register(self): # 注入两个manager self._flow_meta_mgr = self._("flow_meta_manager") self._job_mgr = self._("job_manager") db = self._("db") self._flow_tpl_dao = FlowTemplateDAO(db) self._user_dao = UserDAO(db) def get_all_flow_meta(self) -> Result: """获得所有注册到容器的flow_meta信息 :return [ { "name": "mooncake_flow", "description": "just a test flow", "jobs": [ { "step_name": "programmer", "job_name": "programmer", "bind_args": { "trigger": { "a": 1, "b": 2, } } } ] }, ] """ all_flow_meta = [ FlowMetaView.from_flow_meta(meta, self._job_mgr) for meta in self._flow_meta_mgr.all() ] return Result.ok(data=all_flow_meta) @check( StrField("flow_meta_name", required=True), ) def get_flow_meta_info(self, flow_meta_name) -> Result: """获取单个的flow_meta详情 """ try: flow_meta = self._flow_meta_mgr.get(flow_meta_name) except ObjectNotFoundException: raise BadReq("flow_meta_not_exist", flow_meta=flow_meta_name) return Result.ok( data=FlowMetaView.from_flow_meta(flow_meta, self._job_mgr)) @check(StrField("flow_meta_name", required=True), StrField("name", required=True, min_len=3, max_len=50, regex="^[a-zA-Z0-9_]+$"), Field("bind_args", type_=dict, required=True, value_of=json.loads), IntField("max_run_instance", required=True, min_=0), IntField("creator", required=True)) def create_flow_template(self, flow_meta_name, name, bind_args, max_run_instance, creator) -> Result: """创建flow_template """ # 获取flow_meta以及检查其存在性 try: flow_meta = self._flow_meta_mgr.get(flow_meta_name) except ObjectNotFoundException: raise BadReq("flow_meta_not_exist", flow_meta=flow_meta_name) # 检查name是否已经存在 if self._flow_tpl_dao.is_name_existed(name): raise BadReq("name_already_exist", name=name) # 检查creator是否存在 creator_user = self._user_dao.query_user_by_id(creator) if not creator_user: raise BadReq("invalid_creator_id", creator=creator) # 校验参数 rs = self._validate_tpl_bind_args(flow_meta, bind_args) if rs.status_code == Result.STATUS_BADREQUEST: return rs bind_args = rs.data created_on = datetime.datetime.now() # 插入吧! flow_template = self._flow_tpl_dao.insert_flow_template( flow_meta_name, name, json.dumps(bind_args), max_run_instance, creator, created_on) log.acolyte.info("New flow template created, {}".format( to_json(flow_template))) # 返回刚创建的View return Result.ok(data=FlowTemplateView.from_flow_template( flow_template, creator_user)) # 校验template的绑定参数 def _validate_tpl_bind_args(self, flow_meta, bind_args): new_bind_args = {} for job_ref in flow_meta.jobs: job = self._job_mgr.get(job_ref.job_name) new_bind_args[job.name] = {} for event, job_arg_declares in job.job_args.items(): new_bind_args[job.name][event] = {} for job_arg_declare in job_arg_declares: try: bind_value = get_from_nested_dict( bind_args, job.name, event, job_arg_declare.name) # const 类型的参数不允许被绑定 if job_arg_declare.mark == JobArg.MARK_CONST: continue # 如果为None并且是auto类型,那么可以在此不检查 if bind_value is None and \ job_arg_declare.mark == JobArg.MARK_AUTO: continue # 执行校验并替换新值 new_value = job_arg_declare.field_info(bind_value) new_bind_args[job.name][event][ job_arg_declare.name] = new_value except InvalidFieldException as e: field_name = "{job_name}_{event}_{arg_name}".format( job_name=job.name, event=event, arg_name=job_arg_declare.name) full_reason = "{field_name}_{reason}".format( field_name=field_name, reason=e.reason) # 产生错误消息 loc, _ = locale.getlocale(locale.LC_ALL) msg = default_validate_messages[loc][e.reason] if e.expect is None: msg = msg.format(field_name=field_name) else: msg = msg.format(field_name=field_name, expect=e.expect) return Result.bad_request(full_reason, msg=msg) return Result.ok(data=new_bind_args) def get_all_flow_templates(self): """获取所有的flow_templates列表 """ all_flow_templates = self._flow_tpl_dao.query_all_templates() if not all_flow_templates: return Result.ok(data=[]) users = self._user_dao.query_users_by_id_list( [tpl.creator for tpl in all_flow_templates], to_dict=True) templates_view = [ FlowTemplateView.from_flow_template(tpl, users[tpl.creator]) for tpl in all_flow_templates ] return Result.ok(data=templates_view) def get_flow_template(self, flow_template_id: int): """获取单个的flow_template详情 """ pass def get_flow_instance_by_status(self, status, offsert_id, limit, order): """根据当前的状态获取flow instance列表 """ pass def get_flow_instance_by_template(self, template_id, offet_id, limit, order): """依据flow_template来查询flow实例 """ pass def get_flow_instance_by_template_and_status(self, template_id, status, offset_id, limit, order): """依据flow_template和status来查询flow实例 """ pass def get_flow_instance(self, flow_instance_id): """根据id获取flow实例 """ pass