def test_execute_fork(self):
logger = create_logger("gf", (stdout_handler(),))
worklist = (
Job("add_one"),
MainThreadFork("fork"),
Job("add_three", args=("$add_one.result",)),
MainThreadJoin("join", join=lambda ctx: ctx["add_three"])
)
workflow = Workflow(worklist, None, self.plugin_mgr, logger=logger)
end = workflow.execute(args={"add_one": [1]})
self.assertIsNone(end.result) # 写入的是thread local上下文，而非全局上下文
def save_to_parrent(ctx):
ctx.parrent["add_three"] = ctx["add_three"]
worklist = (
Job("add_one"),
MainThreadFork("fork"),
Job("add_three", args=("$add_one.result",)),
Job("collect", caller=save_to_parrent),
MainThreadJoin("join", join=lambda ctx: ctx["add_three"])
)
workflow = Workflow(worklist, None, self.plugin_mgr, logger=logger)
end = workflow.execute(args={"add_one": [1]})
self.assertEquals(end.result, 5)
def _get_workflow_engine(config, workflow_module,
workflow_options, current_env):
"""获取工作流执行引擎"""
# 获取工作单元列表
workflow_list = getattr(workflow_module, "workflow", None)
if not workflow_list:
show_msg_and_exit(u"工作流单元列表不能为空")
if isinstance(workflow_list, types.FunctionType):
workflow_list = workflow_list(workflow_options)
if not workflow_list:
show_msg_and_exit(u"工作流单元列表不能为空")
# 获取并初始化插件管理器
global plugin_manager
plugin_manager = getattr(
workflow_module, "plugin_manager", DEFAULT_PLUGIN_MANAGER)
global plugin_names
plugin_names = set(work_unit.plugin_name for work_unit in workflow_list
if work_unit.unittype == "job" and
work_unit.plugin_name)
# 额外插件列表
extra_plugin_names = getattr(workflow_module, "plugins", tuple())
for plugin_name in extra_plugin_names:
plugin_names.add(plugin_name)
plugin_manager.sys_prepare(config, *plugin_names)
# 获取logger
logger = getattr(workflow_module, "logger", None)
if isinstance(logger, types.FunctionType):
logger = logger(workflow_options)
logger_level = getattr(workflow_module, "logger_level", logging.INFO)
if isinstance(logger_level, types.FunctionType):
logger_level = logger_level(workflow_options)
if isinstance(logger_level, str):
logger_level = get_logger_level_by_name(logger_level)
if logger is None:
logger = create_logger("girlfriend", (stdout_handler(),),
level=logger_level)
elif isinstance(logger, str):
logger = create_logger(
"girlfriend", (daily_rotaiting_handler(logger),),
level=logger_level)
workflow_engine = Workflow(
workflow_list, config, plugin_manager, Context, logger)
# 获取监听器列表
listeners = getattr(workflow_module, "listeners", [])
if isinstance(listeners, types.FunctionType):
listeners = listeners(workflow_options)
if not listeners:
listeners = []
for listener in listeners:
workflow_engine.add_listener(listener)
return workflow_engine
def workflow_context(self, config=None, args=None,
plugin_mgr=DEFAULT_PLUGIN_MGR, logger=None):
"""构建测试使用的工作流上下文
"""
config, args = config or {}, args or {}
if logger is None:
logger = create_logger(
"girlfriend", (stdout_handler(),), level=logging.DEBUG)
return Context(None, config, args, plugin_mgr, logger)
def workflow_context(self,
config=None,
args=None,
plugin_mgr=DEFAULT_PLUGIN_MGR,
logger=None):
"""构建测试使用的工作流上下文
"""
config, args = config or {}, args or {}
if logger is None:
logger = create_logger("girlfriend", (stdout_handler(), ),
level=logging.DEBUG)
return Context(None, config, args, plugin_mgr, logger)
def test_execute_fork(self):
logger = create_logger("gf", (stdout_handler(), ))
worklist = (Job("add_one"), MainThreadFork("fork"),
Job("add_three", args=("$add_one.result", )),
MainThreadJoin("join", join=lambda ctx: ctx["add_three"]))
workflow = Workflow(worklist, None, self.plugin_mgr, logger=logger)
end = workflow.execute(args={"add_one": [1]})
self.assertIsNone(end.result) # 写入的是thread local上下文，而非全局上下文
def save_to_parrent(ctx):
ctx.parrent["add_three"] = ctx["add_three"]
worklist = (Job("add_one"), MainThreadFork("fork"),
Job("add_three", args=("$add_one.result", )),
Job("collect", caller=save_to_parrent),
MainThreadJoin("join", join=lambda ctx: ctx["add_three"]))
workflow = Workflow(worklist, None, self.plugin_mgr, logger=logger)
end = workflow.execute(args={"add_one": [1]})
self.assertEquals(end.result, 5)
def __init__(self, workflow_list, config=None,
plugin_mgr=plugin_manager, context_factory=Context,
logger=None, parrent_context=None, thread_id=None):
"""
:param workflow_list 工作单元列表
:param config 配置数据
:param plugin_mgr 插件管理器，默认是plugin自带的entry_points管理器
:param context_factory 上下文工厂，必须具有config, args, plugin_mgr, parent
这四个约定的参数
:param logger 日志对象
"""
self._workflow_list = workflow_list
self._config = config or Config()
self._plugin_manager = plugin_mgr
self._units = {} # 以名称为Key的工作流单元引用字典
for idx, unit in enumerate(self._workflow_list):
if unit.name in self._units:
raise WorkflowUnitExistedException(unit.name)
self._units[unit.name] = unit
if unit.unittype == "job" or unit.unittype == "join":
# 如果未指定goto，那么goto的默认值是下一个节点
if unit.goto is None:
if idx < len(self._workflow_list) - 1:
unit.goto = self._workflow_list[idx + 1].name
else:
unit.goto = "end"
elif unit.unittype == "fork":
# 自动设置起始节点
if unit.start_point is None:
if idx < len(self._workflow_list) - 1:
unit.start_point = self._workflow_list[idx + 1].name
else:
raise InvalidArgumentException(
u"Fork单元 '{}' 必须指定一个有效的start_point参数")
# 设置下一步运行的goto节点，如果未指定，则设置最近的join
if unit.goto is None:
for i, next_unit in enumerate(
self._workflow_list[idx + 1:], start=idx + 1):
if next_unit.unittype == "join":
unit.goto = next_unit.name
break
else:
raise InvalidArgumentException(
u"Fork单元 '{}' 必须使用一个有效的goto跳转到Join"
.format(unit.name))
# 自动设置结束节点
if unit.end_point is None:
for i, next_unit in enumerate(
self._workflow_list[idx + 1:], start=idx + 1):
if (
next_unit.unittype == "join" and
next_unit.name == unit.goto
):
# join unit前一个元素
unit.end_point = self._workflow_list[i - 1].name
break
else:
raise InvalidArgumentException(
u"Fork单元 '{}' 必须指定一个有效的end_point参数".format(
unit.name)
)
self._context_factory = context_factory
self._listeners = []
# 创建logger
if logger is None:
self._logger = create_logger("girlfriend", (stdout_handler(),))
else:
self._logger = logger
self._parrent_context = parrent_context
self._thread_id = thread_id
def __init__(self,
workflow_list,
config=None,
plugin_mgr=plugin_manager,
context_factory=Context,
logger=None,
parrent_context=None,
thread_id=None):
"""
:param workflow_list 工作单元列表
:param config 配置数据
:param plugin_mgr 插件管理器，默认是plugin自带的entry_points管理器
:param context_factory 上下文工厂，必须具有config, args, plugin_mgr, parent
这四个约定的参数
:param logger 日志对象
"""
self._workflow_list = workflow_list
self._config = config or Config()
self._plugin_manager = plugin_mgr
self._units = {} # 以名称为Key的工作流单元引用字典
for idx, unit in enumerate(self._workflow_list):
if unit.name in self._units:
raise WorkflowUnitExistedException(unit.name)
self._units[unit.name] = unit
if unit.unittype == "job" or unit.unittype == "join":
# 如果未指定goto，那么goto的默认值是下一个节点
if unit.goto is None:
if idx < len(self._workflow_list) - 1:
unit.goto = self._workflow_list[idx + 1].name
else:
unit.goto = "end"
elif unit.unittype == "fork":
# 自动设置起始节点
if unit.start_point is None:
if idx < len(self._workflow_list) - 1:
unit.start_point = self._workflow_list[idx + 1].name
else:
raise InvalidArgumentException(
u"Fork单元 '{}' 必须指定一个有效的start_point参数")
# 设置下一步运行的goto节点，如果未指定，则设置最近的join
if unit.goto is None:
for i, next_unit in enumerate(self._workflow_list[idx +
1:],
start=idx + 1):
if next_unit.unittype == "join":
unit.goto = next_unit.name
break
else:
raise InvalidArgumentException(
u"Fork单元 '{}' 必须使用一个有效的goto跳转到Join".format(
unit.name))
# 自动设置结束节点
if unit.end_point is None:
for i, next_unit in enumerate(self._workflow_list[idx +
1:],
start=idx + 1):
if (next_unit.unittype == "join"
and next_unit.name == unit.goto):
# join unit前一个元素
unit.end_point = self._workflow_list[i - 1].name
break
else:
raise InvalidArgumentException(
u"Fork单元 '{}' 必须指定一个有效的end_point参数".format(
unit.name))
self._context_factory = context_factory
self._listeners = []
# 创建logger
if logger is None:
self._logger = create_logger("girlfriend", (stdout_handler(), ))
else:
self._logger = logger
self._parrent_context = parrent_context
self._thread_id = thread_id