def _get_workflow_engine(config, workflow_module, workflow_options, current_env): """获取工作流执行引擎""" # 获取工作单元列表 workflow_list = getattr(workflow_module, "workflow", None) if not workflow_list: show_msg_and_exit(u"工作流单元列表不能为空") if isinstance(workflow_list, types.FunctionType): workflow_list = workflow_list(workflow_options) if not workflow_list: show_msg_and_exit(u"工作流单元列表不能为空") # 获取并初始化插件管理器 global plugin_manager plugin_manager = getattr( workflow_module, "plugin_manager", DEFAULT_PLUGIN_MANAGER) global plugin_names plugin_names = set(work_unit.plugin_name for work_unit in workflow_list if work_unit.unittype == "job" and work_unit.plugin_name) # 额外插件列表 extra_plugin_names = getattr(workflow_module, "plugins", tuple()) for plugin_name in extra_plugin_names: plugin_names.add(plugin_name) plugin_manager.sys_prepare(config, *plugin_names) # 获取logger logger = getattr(workflow_module, "logger", None) if isinstance(logger, types.FunctionType): logger = logger(workflow_options) logger_level = getattr(workflow_module, "logger_level", logging.INFO) if isinstance(logger_level, types.FunctionType): logger_level = logger_level(workflow_options) if isinstance(logger_level, str): logger_level = get_logger_level_by_name(logger_level) if logger is None: logger = create_logger("girlfriend", (stdout_handler(),), level=logger_level) elif isinstance(logger, str): logger = create_logger( "girlfriend", (daily_rotaiting_handler(logger),), level=logger_level) workflow_engine = Workflow( workflow_list, config, plugin_manager, Context, logger) # 获取监听器列表 listeners = getattr(workflow_module, "listeners", []) if isinstance(listeners, types.FunctionType): listeners = listeners(workflow_options) if not listeners: listeners = [] for listener in listeners: workflow_engine.add_listener(listener) return workflow_engine
def _get_current_env(workflow_module): env_list = getattr(workflow_module, "env", None) if env_list is None: return Env.test_env() current_env_name = TOOLS_OPTIONS.environ for env in env_list: if env.name == current_env_name: return env show_msg_and_exit(u"找不到目标环境:'{}'".format(current_env_name))
def _get_current_env(workflow_module): env_list = getattr(workflow_module, "env", None) if env_list is None: return Env.test_env() current_env_name = TOOLS_OPTIONS.environ for env in env_list: if env.name == current_env_name: return env show_msg_and_exit(u"找不到目标环境:'{}'".format(current_env_name))
def workflow(options): # 检查配置文件 if not options.task: show_msg_and_exit(u"必须指定一个任务描述文件,文件格式见帮助") if not os.path.exists(options.task): show_msg_and_exit(u"任务描述文件 '{}' 不存在".format(options.task)) return ( # 读取json格式的任务描述文件并保存到上下文 Job("read_json", args=(JSONR(options.task, "block", result_wrapper=_task, variable="task"), )), # 执行SQL语句 Job("orm_query", args=_gen_orm_query_args), # 决定是否打印表格到终端 Decision( "need_print", lambda ctx: "print_table" if options.print_tables else "need_write_excel"), # print tables Job("print_table", args="orm_query.result"), # 决定是否要输出Excel Decision( "need_write_excel", lambda ctx: "write_excel" if ctx["task"].get("workbooks") else "need_send_mail", ), # 输出Excel Job("write_excel", args=_gen_excel_args), # 是否发送Email Decision( "need_send_mail", lambda ctx: "need_render_html" if ctx["task"].get("mail") else "end"), # 是否需要渲染html表格 Decision( "need_render_html", lambda ctx: "html_table" if ctx["task"]["mail"].get("tables") else "send_mail"), # 渲染html表格 Job("html_table", args=lambda ctx: [ HTMLTable(ctx["orm_query.result"][idx], property=ctx["task"]["mail"].get("table_property")) for idx in ctx["task"]["mail"]["tables"] ]), # 发送邮件了! Job("send_mail", args=_gen_send_mail_args))
def _load_module(): """加载工作流模块""" module_path = TOOLS_OPTIONS.module if not module_path: show_msg_and_exit(u"必须使用-m参数指定一个工作流描述模块") if module_path.startswith(":"): module_path = module_path[1:] for ep in pkg_resources.iter_entry_points("girlfriend.workflow"): if ep.name == module_path: return ep.load() show_msg_and_exit(u"找不到工作流模块 '{}'".format(module_path)) # 以.py结尾,那么按照python文件的形式进行加载 elif module_path.endswith(".py"): if not os.path.exists(module_path): show_msg_and_exit(u"找不到工作流描述文件 '{}'".format(module_path)) return imp.load_source("workflow", module_path) # 按照模块名称方式进行加载 else: try: return __import__(module_path, fromlist=[""]) except ImportError: show_msg_and_exit(u"找不到工作流模块 '{}'".format(module_path))
def _load_module(): """加载工作流模块""" module_path = TOOLS_OPTIONS.module if not module_path: show_msg_and_exit(u"必须使用-m参数指定一个工作流描述模块") if module_path.startswith(":"): module_path = module_path[1:] for ep in pkg_resources.iter_entry_points("girlfriend.workflow"): if ep.name == module_path: return ep.load() show_msg_and_exit(u"找不到工作流模块 '{}'".format(module_path)) # 以.py结尾,那么按照python文件的形式进行加载 elif module_path.endswith(".py"): if not os.path.exists(module_path): show_msg_and_exit(u"找不到工作流描述文件 '{}'".format(module_path)) return imp.load_source("workflow", module_path) # 按照模块名称方式进行加载 else: try: return __import__(module_path, fromlist=[""]) except ImportError: show_msg_and_exit(u"找不到工作流模块 '{}'".format(module_path))
def main(): options = parse_cmd_args() tpl_module = load_module(options.template, entry_point="girlfriend.code_template") if tpl_module is None: show_msg_and_exit(u"找不到模板模块 '{}'".format(options.template)) tpl_options = None tpl_cmd_parser = getattr(tpl_module, "cmd_parser", None) if tpl_cmd_parser is not None: if options.show_args: print_help(tpl_cmd_parser) return tpl_options = tpl_cmd_parser.parse_known_args(sys.argv[1:])[0] tpl_module.gen(options.path, tpl_options)
def main(): options = parse_cmd_args() if not options.template: show_msg_and_exit(u"您必须指定一个生成模板,比如 -t :workflow") tpl_module = load_module(options.template, entry_point="girlfriend.code_template") if tpl_module is None: show_msg_and_exit(u"找不到模板模块 '{}'".format(options.template)) tpl_options = None tpl_cmd_parser = getattr(tpl_module, "cmd_parser", None) if tpl_cmd_parser is not None: if options.show_args: print_help(tpl_cmd_parser) return tpl_options = tpl_cmd_parser.parse_known_args(sys.argv[1:])[0] tpl_module.gen(options.path, tpl_options)
def main(): cmd_options = _parse_cmd_args() if os.path.exists(cmd_options.data_file): answer = raw_input(u"数据文件 '{}' 已经存在,是否覆盖?(y/n)".format( cmd_options.data_file).encode("utf-8")) if answer == 'y': os.remove(cmd_options.data_file) else: exit(0) with sqlite3.connect(cmd_options.data_file) as conn: # 创建表 conn.executescript(CREATE_USER_TABLE) conn.executescript(CREATE_CAT_TABLE) # 填充数据 cursor = conn.cursor() cursor.executemany(INSERT_USER, USER_DATA) cursor.executemany(INSERT_CAT, CAT_DATA) show_msg_and_exit(u"测试数据文件 '{}' 已经创建完毕!".format(cmd_options.data_file), "green")
def main(): cmd_options = _parse_cmd_args() if os.path.exists(cmd_options.data_file): answer = raw_input( u"数据文件 '{}' 已经存在,是否覆盖?(y/n)".format( cmd_options.data_file).encode("utf-8")) if answer == 'y': os.remove(cmd_options.data_file) else: exit(0) with sqlite3.connect(cmd_options.data_file) as conn: # 创建表 conn.executescript(CREATE_USER_TABLE) conn.executescript(CREATE_CAT_TABLE) # 填充数据 cursor = conn.cursor() cursor.executemany(INSERT_USER, USER_DATA) cursor.executemany(INSERT_CAT, CAT_DATA) show_msg_and_exit(u"测试数据文件 '{}' 已经创建完毕!".format(cmd_options.data_file), "green")
def main(): cmd_args = parse_cmd_args() try: if cmd_args.f: gen_config_file(cmd_args.f) script.show_msg_and_exit(u"已经成功创建配置文件'{}'".format(cmd_args.f), "green") else: gen_home_workspace() print WELCOME script.show_msg_and_exit( u"默认工作目录以及默认配置文件创建成功,请查看 '{}'".format(HOME_WORKSPACE), "green") except GirlFriendBizException as biz_e: script.show_msg_and_exit(unicode(biz_e), "yellow") except Exception as sys_e: script.show_traceback_and_exit(unicode(sys_e))
def main(): cmd_args = parse_cmd_args() try: if cmd_args.f: gen_config_file(cmd_args.f) script.show_msg_and_exit( u"已经成功创建配置文件'{}'".format(cmd_args.f), "green") else: gen_home_workspace() print WELCOME script.show_msg_and_exit( u"默认工作目录以及默认配置文件创建成功,请查看 '{}'".format(HOME_WORKSPACE), "green") except GirlFriendBizException as biz_e: script.show_msg_and_exit(unicode(biz_e), "yellow") except Exception as sys_e: script.show_traceback_and_exit(unicode(sys_e))
def workflow(options): # 检查配置文件 if not options.task: show_msg_and_exit(u"必须指定一个任务描述文件,文件格式见帮助") if not os.path.exists(options.task): show_msg_and_exit(u"任务描述文件 '{}' 不存在".format(options.task)) return ( # 读取json格式的任务描述文件并保存到上下文 Job( "read_json", args=( JSONR(options.task, "block", result_wrapper=_task, variable="task"), ) ), # 执行SQL语句 Job( "orm_query", args=_gen_orm_query_args ), # 决定是否打印表格到终端 Decision( "need_print", lambda ctx: "print_table" if options.print_tables else "need_write_excel" ), # print tables Job( "print_table", args="orm_query.result" ), # 决定是否要输出Excel Decision( "need_write_excel", lambda ctx: "write_excel" if ctx["task"].get("workbooks") else "need_send_mail", ), # 输出Excel Job( "write_excel", args=_gen_excel_args ), # 是否发送Email Decision( "need_send_mail", lambda ctx: "need_render_html" if ctx["task"].get("mail") else "end" ), # 是否需要渲染html表格 Decision( "need_render_html", lambda ctx: "html_table" if ctx["task"]["mail"].get("tables") else "send_mail" ), # 渲染html表格 Job( "html_table", args=lambda ctx: [HTMLTable( ctx["orm_query.result"][idx], property=ctx["task"]["mail"].get("table_property")) for idx in ctx["task"]["mail"]["tables"]] ), # 发送邮件了! Job( "send_mail", args=_gen_send_mail_args ) )