def _get_workflow_engine(config, workflow_module,
workflow_options, current_env):
"""获取工作流执行引擎"""
# 获取工作单元列表
workflow_list = getattr(workflow_module, "workflow", None)
if not workflow_list:
show_msg_and_exit(u"工作流单元列表不能为空")
if isinstance(workflow_list, types.FunctionType):
workflow_list = workflow_list(workflow_options)
if not workflow_list:
show_msg_and_exit(u"工作流单元列表不能为空")
# 获取并初始化插件管理器
global plugin_manager
plugin_manager = getattr(
workflow_module, "plugin_manager", DEFAULT_PLUGIN_MANAGER)
global plugin_names
plugin_names = set(work_unit.plugin_name for work_unit in workflow_list
if work_unit.unittype == "job" and
work_unit.plugin_name)
# 额外插件列表
extra_plugin_names = getattr(workflow_module, "plugins", tuple())
for plugin_name in extra_plugin_names:
plugin_names.add(plugin_name)
plugin_manager.sys_prepare(config, *plugin_names)
# 获取logger
logger = getattr(workflow_module, "logger", None)
if isinstance(logger, types.FunctionType):
logger = logger(workflow_options)
logger_level = getattr(workflow_module, "logger_level", logging.INFO)
if isinstance(logger_level, types.FunctionType):
logger_level = logger_level(workflow_options)
if isinstance(logger_level, str):
logger_level = get_logger_level_by_name(logger_level)
if logger is None:
logger = create_logger("girlfriend", (stdout_handler(),),
level=logger_level)
elif isinstance(logger, str):
logger = create_logger(
"girlfriend", (daily_rotaiting_handler(logger),),
level=logger_level)
workflow_engine = Workflow(
workflow_list, config, plugin_manager, Context, logger)
# 获取监听器列表
listeners = getattr(workflow_module, "listeners", [])
if isinstance(listeners, types.FunctionType):
listeners = listeners(workflow_options)
if not listeners:
listeners = []
for listener in listeners:
workflow_engine.add_listener(listener)
return workflow_engine
def _get_current_env(workflow_module):
env_list = getattr(workflow_module, "env", None)
if env_list is None:
return Env.test_env()
current_env_name = TOOLS_OPTIONS.environ
for env in env_list:
if env.name == current_env_name:
return env
show_msg_and_exit(u"找不到目标环境：'{}'".format(current_env_name))
def _get_current_env(workflow_module):
env_list = getattr(workflow_module, "env", None)
if env_list is None:
return Env.test_env()
current_env_name = TOOLS_OPTIONS.environ
for env in env_list:
if env.name == current_env_name:
return env
show_msg_and_exit(u"找不到目标环境：'{}'".format(current_env_name))
def workflow(options):
# 检查配置文件
if not options.task:
show_msg_and_exit(u"必须指定一个任务描述文件，文件格式见帮助")
if not os.path.exists(options.task):
show_msg_and_exit(u"任务描述文件 '{}' 不存在".format(options.task))
return (
# 读取json格式的任务描述文件并保存到上下文
Job("read_json",
args=(JSONR(options.task,
"block",
result_wrapper=_task,
variable="task"), )),
# 执行SQL语句
Job("orm_query", args=_gen_orm_query_args),
# 决定是否打印表格到终端
Decision(
"need_print", lambda ctx: "print_table"
if options.print_tables else "need_write_excel"),
# print tables
Job("print_table", args="orm_query.result"),
# 决定是否要输出Excel
Decision(
"need_write_excel",
lambda ctx: "write_excel"
if ctx["task"].get("workbooks") else "need_send_mail",
),
# 输出Excel
Job("write_excel", args=_gen_excel_args),
# 是否发送Email
Decision(
"need_send_mail", lambda ctx: "need_render_html"
if ctx["task"].get("mail") else "end"),
# 是否需要渲染html表格
Decision(
"need_render_html", lambda ctx: "html_table"
if ctx["task"]["mail"].get("tables") else "send_mail"),
# 渲染html表格
Job("html_table",
args=lambda ctx: [
HTMLTable(ctx["orm_query.result"][idx],
property=ctx["task"]["mail"].get("table_property"))
for idx in ctx["task"]["mail"]["tables"]
]),
# 发送邮件了!
Job("send_mail", args=_gen_send_mail_args))
def _load_module():
"""加载工作流模块"""
module_path = TOOLS_OPTIONS.module
if not module_path:
show_msg_and_exit(u"必须使用-m参数指定一个工作流描述模块")
if module_path.startswith(":"):
module_path = module_path[1:]
for ep in pkg_resources.iter_entry_points("girlfriend.workflow"):
if ep.name == module_path:
return ep.load()
show_msg_and_exit(u"找不到工作流模块 '{}'".format(module_path))
# 以.py结尾，那么按照python文件的形式进行加载
elif module_path.endswith(".py"):
if not os.path.exists(module_path):
show_msg_and_exit(u"找不到工作流描述文件 '{}'".format(module_path))
return imp.load_source("workflow", module_path)
# 按照模块名称方式进行加载
else:
try:
return __import__(module_path, fromlist=[""])
except ImportError:
show_msg_and_exit(u"找不到工作流模块 '{}'".format(module_path))
def _load_module():
"""加载工作流模块"""
module_path = TOOLS_OPTIONS.module
if not module_path:
show_msg_and_exit(u"必须使用-m参数指定一个工作流描述模块")
if module_path.startswith(":"):
module_path = module_path[1:]
for ep in pkg_resources.iter_entry_points("girlfriend.workflow"):
if ep.name == module_path:
return ep.load()
show_msg_and_exit(u"找不到工作流模块 '{}'".format(module_path))
# 以.py结尾，那么按照python文件的形式进行加载
elif module_path.endswith(".py"):
if not os.path.exists(module_path):
show_msg_and_exit(u"找不到工作流描述文件 '{}'".format(module_path))
return imp.load_source("workflow", module_path)
# 按照模块名称方式进行加载
else:
try:
return __import__(module_path, fromlist=[""])
except ImportError:
show_msg_and_exit(u"找不到工作流模块 '{}'".format(module_path))
def main():
options = parse_cmd_args()
tpl_module = load_module(options.template,
entry_point="girlfriend.code_template")
if tpl_module is None:
show_msg_and_exit(u"找不到模板模块 '{}'".format(options.template))
tpl_options = None
tpl_cmd_parser = getattr(tpl_module, "cmd_parser", None)
if tpl_cmd_parser is not None:
if options.show_args:
print_help(tpl_cmd_parser)
return
tpl_options = tpl_cmd_parser.parse_known_args(sys.argv[1:])[0]
tpl_module.gen(options.path, tpl_options)
def main():
options = parse_cmd_args()
if not options.template:
show_msg_and_exit(u"您必须指定一个生成模板，比如 -t :workflow")
tpl_module = load_module(options.template,
entry_point="girlfriend.code_template")
if tpl_module is None:
show_msg_and_exit(u"找不到模板模块 '{}'".format(options.template))
tpl_options = None
tpl_cmd_parser = getattr(tpl_module, "cmd_parser", None)
if tpl_cmd_parser is not None:
if options.show_args:
print_help(tpl_cmd_parser)
return
tpl_options = tpl_cmd_parser.parse_known_args(sys.argv[1:])[0]
tpl_module.gen(options.path, tpl_options)
def main():
cmd_options = _parse_cmd_args()
if os.path.exists(cmd_options.data_file):
answer = raw_input(u"数据文件 '{}' 已经存在，是否覆盖？(y/n)".format(
cmd_options.data_file).encode("utf-8"))
if answer == 'y':
os.remove(cmd_options.data_file)
else:
exit(0)
with sqlite3.connect(cmd_options.data_file) as conn:
# 创建表
conn.executescript(CREATE_USER_TABLE)
conn.executescript(CREATE_CAT_TABLE)
# 填充数据
cursor = conn.cursor()
cursor.executemany(INSERT_USER, USER_DATA)
cursor.executemany(INSERT_CAT, CAT_DATA)
show_msg_and_exit(u"测试数据文件 '{}' 已经创建完毕!".format(cmd_options.data_file),
"green")
def main():
cmd_options = _parse_cmd_args()
if os.path.exists(cmd_options.data_file):
answer = raw_input(
u"数据文件 '{}' 已经存在，是否覆盖？(y/n)".format(
cmd_options.data_file).encode("utf-8"))
if answer == 'y':
os.remove(cmd_options.data_file)
else:
exit(0)
with sqlite3.connect(cmd_options.data_file) as conn:
# 创建表
conn.executescript(CREATE_USER_TABLE)
conn.executescript(CREATE_CAT_TABLE)
# 填充数据
cursor = conn.cursor()
cursor.executemany(INSERT_USER, USER_DATA)
cursor.executemany(INSERT_CAT, CAT_DATA)
show_msg_and_exit(u"测试数据文件 '{}' 已经创建完毕!".format(cmd_options.data_file),
"green")
def main():
cmd_args = parse_cmd_args()
try:
if cmd_args.f:
gen_config_file(cmd_args.f)
script.show_msg_and_exit(u"已经成功创建配置文件'{}'".format(cmd_args.f),
"green")
else:
gen_home_workspace()
print WELCOME
script.show_msg_and_exit(
u"默认工作目录以及默认配置文件创建成功，请查看 '{}'".format(HOME_WORKSPACE), "green")
except GirlFriendBizException as biz_e:
script.show_msg_and_exit(unicode(biz_e), "yellow")
except Exception as sys_e:
script.show_traceback_and_exit(unicode(sys_e))
def main():
cmd_args = parse_cmd_args()
try:
if cmd_args.f:
gen_config_file(cmd_args.f)
script.show_msg_and_exit(
u"已经成功创建配置文件'{}'".format(cmd_args.f), "green")
else:
gen_home_workspace()
print WELCOME
script.show_msg_and_exit(
u"默认工作目录以及默认配置文件创建成功，请查看 '{}'".format(HOME_WORKSPACE),
"green")
except GirlFriendBizException as biz_e:
script.show_msg_and_exit(unicode(biz_e), "yellow")
except Exception as sys_e:
script.show_traceback_and_exit(unicode(sys_e))
def workflow(options):
# 检查配置文件
if not options.task:
show_msg_and_exit(u"必须指定一个任务描述文件，文件格式见帮助")
if not os.path.exists(options.task):
show_msg_and_exit(u"任务描述文件 '{}' 不存在".format(options.task))
return (
# 读取json格式的任务描述文件并保存到上下文
Job(
"read_json",
args=(
JSONR(options.task, "block",
result_wrapper=_task, variable="task"),
)
),
# 执行SQL语句
Job(
"orm_query",
args=_gen_orm_query_args
),
# 决定是否打印表格到终端
Decision(
"need_print",
lambda ctx: "print_table"
if options.print_tables else "need_write_excel"
),
# print tables
Job(
"print_table",
args="orm_query.result"
),
# 决定是否要输出Excel
Decision(
"need_write_excel",
lambda ctx: "write_excel"
if ctx["task"].get("workbooks") else "need_send_mail",
),
# 输出Excel
Job(
"write_excel",
args=_gen_excel_args
),
# 是否发送Email
Decision(
"need_send_mail",
lambda ctx: "need_render_html"
if ctx["task"].get("mail") else "end"
),
# 是否需要渲染html表格
Decision(
"need_render_html",
lambda ctx: "html_table"
if ctx["task"]["mail"].get("tables") else "send_mail"
),
# 渲染html表格
Job(
"html_table",
args=lambda ctx: [HTMLTable(
ctx["orm_query.result"][idx],
property=ctx["task"]["mail"].get("table_property"))
for idx in ctx["task"]["mail"]["tables"]]
),
# 发送邮件了!
Job(
"send_mail",
args=_gen_send_mail_args
)
)