def __call__(self):
try:
sub_workflow = Workflow(
self.units,
config=self.parrent_context.config,
plugin_mgr=self.parrent_context.plugin_mgr,
context_factory=self.context_factory,
logger=self.parrent_context.logger,
parrent_context=self.parrent_context,
thread_id=self.thread_id
)
end = sub_workflow.execute(
None, self.start_point, self.end_point)
self.parrent_context[
"_fork.result"][self.thread_id] = end
except Exception:
exc_type, exc_value, tb = sys.exc_info()
self.parrent_context.logger.exception(
u"子线程 - '{}' 发生了异常".format(self.thread_id))
self.parrent_context[
"_fork.result"][self.thread_id] = ErrorEnd(
exc_type, exc_value, tb)
finally:
self.parrent_context[
"_fork.count_down_latch"].count_down()
def test_execute_fork(self):
logger = create_logger("gf", (stdout_handler(),))
worklist = (
Job("add_one"),
MainThreadFork("fork"),
Job("add_three", args=("$add_one.result",)),
MainThreadJoin("join", join=lambda ctx: ctx["add_three"])
)
workflow = Workflow(worklist, None, self.plugin_mgr, logger=logger)
end = workflow.execute(args={"add_one": [1]})
self.assertIsNone(end.result) # 写入的是thread local上下文，而非全局上下文
def save_to_parrent(ctx):
ctx.parrent["add_three"] = ctx["add_three"]
worklist = (
Job("add_one"),
MainThreadFork("fork"),
Job("add_three", args=("$add_one.result",)),
Job("collect", caller=save_to_parrent),
MainThreadJoin("join", join=lambda ctx: ctx["add_three"])
)
workflow = Workflow(worklist, None, self.plugin_mgr, logger=logger)
end = workflow.execute(args={"add_one": [1]})
self.assertEquals(end.result, 5)
def setUp(self):
wf = Workflow([
Job(name="add", caller=lambda ctx, x, y: x + y),
Job(name="divide", caller=lambda ctx, x, y: x / y)
])
wf.add_listener(PicklePersistListener(self._dump_to))
self.wf = wf
def _get_workflow_engine(config, workflow_module,
workflow_options, current_env):
"""获取工作流执行引擎"""
# 获取工作单元列表
workflow_list = getattr(workflow_module, "workflow", None)
if not workflow_list:
show_msg_and_exit(u"工作流单元列表不能为空")
if isinstance(workflow_list, types.FunctionType):
workflow_list = workflow_list(workflow_options)
if not workflow_list:
show_msg_and_exit(u"工作流单元列表不能为空")
# 获取并初始化插件管理器
global plugin_manager
plugin_manager = getattr(
workflow_module, "plugin_manager", DEFAULT_PLUGIN_MANAGER)
global plugin_names
plugin_names = set(work_unit.plugin_name for work_unit in workflow_list
if work_unit.unittype == "job" and
work_unit.plugin_name)
# 额外插件列表
extra_plugin_names = getattr(workflow_module, "plugins", tuple())
for plugin_name in extra_plugin_names:
plugin_names.add(plugin_name)
plugin_manager.sys_prepare(config, *plugin_names)
# 获取logger
logger = getattr(workflow_module, "logger", None)
if isinstance(logger, types.FunctionType):
logger = logger(workflow_options)
logger_level = getattr(workflow_module, "logger_level", logging.INFO)
if isinstance(logger_level, types.FunctionType):
logger_level = logger_level(workflow_options)
if isinstance(logger_level, str):
logger_level = get_logger_level_by_name(logger_level)
if logger is None:
logger = create_logger("girlfriend", (stdout_handler(),),
level=logger_level)
elif isinstance(logger, str):
logger = create_logger(
"girlfriend", (daily_rotaiting_handler(logger),),
level=logger_level)
workflow_engine = Workflow(
workflow_list, config, plugin_manager, Context, logger)
# 获取监听器列表
listeners = getattr(workflow_module, "listeners", [])
if isinstance(listeners, types.FunctionType):
listeners = listeners(workflow_options)
if not listeners:
listeners = []
for listener in listeners:
workflow_engine.add_listener(listener)
return workflow_engine
def test_execute_with_invalid_args(self):
# 测试错误请求参数
worklist = (Job("add_four"), )
workflow = Workflow(worklist, None, self.plugin_mgr)
end = workflow.execute(args={"add_four": (1, )})
self.assertEquals(end.result, 5)
end = workflow.execute(args={"add_four": (-1, )})
self.assertEquals(end.status, End.STATUS_BAD_REQUEST)
def test_add_listener(self):
listener = ListenerA(0)
worklist = (Job("add_one"), Job("add_two", args=("$add_one.result", )),
Decision(
"decide", lambda ctx: "add_three"
if ctx["add_two.result"] > 10 else "division"),
Job("add_three", args=("$add_two.result", )),
OkEnd("add_three_end",
execute=lambda ctx: ctx["add_three"]),
Job("division", args=("$add_two.result", 2)))
workflow = Workflow(worklist, None, self.plugin_mgr)
workflow.add_listener(listener)
workflow.add_listener(ListenerB)
end = workflow.execute(args={"add_one": (1, )})
self.assertEquals(end.result, 2)
self.assertEquals(listener.a, -4)
listener.a = 0
end = workflow.execute(args={"add_one": (10, )})
self.assertEquals(end.result, 16)
self.assertEquals(listener.a, -5)
listener.a = 0
end = workflow.execute(args={
"add_one": (1, ),
"division": ("$add_two.result", 0)
})
self.assertEquals(end.status, End.STATUS_ERROR_HAPPENED)
self.assertEquals(listener.a, -2)
self.assertEquals(end.exc_type, ZeroDivisionError)
def test_execute_with_invalid_args(self):
# 测试错误请求参数
worklist = (
Job("add_four"),
)
workflow = Workflow(worklist, None, self.plugin_mgr)
end = workflow.execute(args={"add_four": (1, )})
self.assertEquals(end.result, 5)
end = workflow.execute(args={"add_four": (-1, )})
self.assertEquals(end.status, End.STATUS_BAD_REQUEST)
def test_execute_with_exception(self):
# 测试异常发生时的情况
worklist = (Job("add_one", args=(5, )),
Job("division", args={"a": "$add_one.result"}))
workflow = Workflow(worklist, None, self.plugin_mgr)
end = workflow.execute(args={"division": {"b": 3}})
self.assertEquals(end.result, 2)
# 除0错误
end = workflow.execute(args={"division": {"b": 0}})
self.assertEquals(end.status, End.STATUS_ERROR_HAPPENED)
self.assertEquals(end.exc_type, ZeroDivisionError)
def test_add_listener(self):
listener = ListenerA(0)
worklist = (
Job("add_one"),
Job("add_two", args=("$add_one.result",)),
Decision("decide", lambda ctx: "add_three"
if ctx["add_two.result"] > 10 else "division"),
Job("add_three", args=("$add_two.result",)),
OkEnd("add_three_end", execute=lambda ctx: ctx["add_three"]),
Job("division", args=("$add_two.result", 2))
)
workflow = Workflow(worklist, None, self.plugin_mgr)
workflow.add_listener(listener)
workflow.add_listener(ListenerB)
end = workflow.execute(args={"add_one": (1, )})
self.assertEquals(end.result, 2)
self.assertEquals(listener.a, -4)
listener.a = 0
end = workflow.execute(args={"add_one": (10, )})
self.assertEquals(end.result, 16)
self.assertEquals(listener.a, -5)
listener.a = 0
end = workflow.execute(args={
"add_one": (1, ),
"division": ("$add_two.result", 0)
})
self.assertEquals(end.status, End.STATUS_ERROR_HAPPENED)
self.assertEquals(listener.a, -2)
self.assertEquals(end.exc_type, ZeroDivisionError)
def test_execute_common(self):
# 测试普通运行时的情况
worklist = (
Job("add_one"),
Job("add_three", args=("$add_one.result",)),
Decision("judge",
lambda ctx: "test_end"
if ctx["add_three"] > 5 else "add_two"),
Job("add_two", args={"num": "$add_three"}, goto="end"),
OkEnd("test_end", execute=lambda ctx: ctx["add_three"]) # 自定义end
)
workflow = Workflow(worklist, None, self.plugin_mgr)
end = workflow.execute(args={"add_one": (1,)})
self.assertEquals(end.result, 7)
end = workflow.execute(args={"add_one": {"num": 10}})
self.assertEquals(end.result, 14)
def test_execute_common(self):
# 测试普通运行时的情况
worklist = (
Job("add_one"),
Job("add_three", args=("$add_one.result", )),
Decision(
"judge", lambda ctx: "test_end"
if ctx["add_three"] > 5 else "add_two"),
Job("add_two", args={"num": "$add_three"}, goto="end"),
OkEnd("test_end", execute=lambda ctx: ctx["add_three"]) # 自定义end
)
workflow = Workflow(worklist, None, self.plugin_mgr)
end = workflow.execute(args={"add_one": (1, )})
self.assertEquals(end.result, 7)
end = workflow.execute(args={"add_one": {"num": 10}})
self.assertEquals(end.result, 14)
def test_execute_with_exception(self):
# 测试异常发生时的情况
worklist = (
Job("add_one", args=(5, )),
Job("division", args={"a": "$add_one.result"})
)
workflow = Workflow(worklist, None, self.plugin_mgr)
end = workflow.execute(args={
"division": {"b": 3}
})
self.assertEquals(end.result, 2)
# 除0错误
end = workflow.execute(args={
"division": {"b": 0}
})
self.assertEquals(end.status, End.STATUS_ERROR_HAPPENED)
self.assertEquals(end.exc_type, ZeroDivisionError)
def __call__(self):
try:
sub_workflow = Workflow(
self.units,
config=self.parrent_context.config,
plugin_mgr=self.parrent_context.plugin_mgr,
context_factory=self.context_factory,
logger=self.parrent_context.logger,
parrent_context=self.parrent_context,
thread_id=self.thread_id)
end = sub_workflow.execute(None, self.start_point,
self.end_point)
self.parrent_context["_fork.result"][self.thread_id] = end
except Exception:
exc_type, exc_value, tb = sys.exc_info()
self.parrent_context.logger.exception(
u"子线程 - '{}' 发生了异常".format(self.thread_id))
self.parrent_context["_fork.result"][
self.thread_id] = ErrorEnd(exc_type, exc_value, tb)
finally:
self.parrent_context["_fork.count_down_latch"].count_down()
def test_execute_fork(self):
logger = create_logger("gf", (stdout_handler(), ))
worklist = (Job("add_one"), MainThreadFork("fork"),
Job("add_three", args=("$add_one.result", )),
MainThreadJoin("join", join=lambda ctx: ctx["add_three"]))
workflow = Workflow(worklist, None, self.plugin_mgr, logger=logger)
end = workflow.execute(args={"add_one": [1]})
self.assertIsNone(end.result) # 写入的是thread local上下文，而非全局上下文
def save_to_parrent(ctx):
ctx.parrent["add_three"] = ctx["add_three"]
worklist = (Job("add_one"), MainThreadFork("fork"),
Job("add_three", args=("$add_one.result", )),
Job("collect", caller=save_to_parrent),
MainThreadJoin("join", join=lambda ctx: ctx["add_three"]))
workflow = Workflow(worklist, None, self.plugin_mgr, logger=logger)
end = workflow.execute(args={"add_one": [1]})
self.assertEquals(end.result, 5)
def test_fork_join(self):
def sleep_task(ctx, task_name, seconds):
ctx.logger.info(
"Enter task '{task_name}' in thread {thread_name}.".format(
task_name=task_name,
thread_name=threading.current_thread().name))
time.sleep(seconds)
ctx.logger.info(
"Leave task '{task_name}' in thread {thread_name}.".format(
task_name=task_name,
thread_name=threading.current_thread().name))
return seconds
def error_happened(ctx):
return 1 / 0
# 正常执行的情况
units = [
Job(name="task_init", caller=sleep_task, args=("init", 1)),
ConcurrentFork(name="fork", thread_num=10),
Job(name="task_first", caller=sleep_task, args=("first", 2)),
Job(name="task_second", caller=sleep_task, args=("second", 3)),
ConcurrentJoin(name="join"),
]
workflow = Workflow(units)
end = workflow.execute()
self.assertEquals(end.result, [3] * 10)
# with customize join
def join_it(ctx, end_list):
return sum(end.result for end in end_list)
sum_units = units[:]
sum_units[-1] = ConcurrentJoin(name="join", join=join_it)
workflow = Workflow(sum_units)
end = workflow.execute()
self.assertEquals(end.result, 30)
# with error happened
error_units = units[:]
error_units[-2] = Job(name="task_second", caller=lambda ctx: 1 / 0)
workflow = Workflow(error_units)
end = workflow.execute()
self.assertEquals(end.status, End.STATUS_ERROR_HAPPENED)
self.assertEquals(end.exc_type, ZeroDivisionError)
# with existed thread pool
pool = ThreadPoolExecutor(5)
existed_pool_units = units[:]
existed_pool_units[1] = ConcurrentFork(name="fork",
thread_num=10,
pool=pool)
workflow = Workflow(existed_pool_units)
end = workflow.execute()
self.assertEquals(end.result, [3] * 10)
def test_fork_join(self):
def sleep_task(ctx, task_name, seconds):
ctx.logger.info(
"Enter task '{task_name}' in thread {thread_name}.".format(
task_name=task_name,
thread_name=threading.current_thread().name
)
)
time.sleep(seconds)
ctx.logger.info(
"Leave task '{task_name}' in thread {thread_name}.".format(
task_name=task_name,
thread_name=threading.current_thread().name
)
)
return seconds
def error_happened(ctx):
return 1 / 0
# 正常执行的情况
units = [
Job(
name="task_init",
caller=sleep_task,
args=("init", 1)
),
ConcurrentFork(
name="fork",
thread_num=10
),
Job(
name="task_first",
caller=sleep_task,
args=("first", 2)
),
Job(
name="task_second",
caller=sleep_task,
args=("second", 3)
),
ConcurrentJoin(
name="join"
),
]
workflow = Workflow(units)
end = workflow.execute()
self.assertEquals(end.result, [3] * 10)
# with customize join
def join_it(ctx, end_list):
return sum(end.result for end in end_list)
sum_units = units[:]
sum_units[-1] = ConcurrentJoin(
name="join",
join=join_it
)
workflow = Workflow(sum_units)
end = workflow.execute()
self.assertEquals(end.result, 30)
# with error happened
error_units = units[:]
error_units[-2] = Job(
name="task_second",
caller=lambda ctx: 1 / 0
)
workflow = Workflow(error_units)
end = workflow.execute()
self.assertEquals(end.status, End.STATUS_ERROR_HAPPENED)
self.assertEquals(end.exc_type, ZeroDivisionError)
# with existed thread pool
pool = ThreadPoolExecutor(5)
existed_pool_units = units[:]
existed_pool_units[1] = ConcurrentFork(
name="fork",
thread_num=10,
pool=pool
)
workflow = Workflow(existed_pool_units)
end = workflow.execute()
self.assertEquals(end.result, [3] * 10)