def main(): conf = get_conf('app') gateway = conf.api.gateway default_conf = get_conf('worker')['default'] register_worker('default', default_conf['max_concurrent'], default_conf['async'], default_conf.get('independent', False), get_ep(default_conf['entrypoint'])) uvicorn.run(app, host=gateway.hostname, port=gateway.port) pool.shutdown()
async def modify_conf( params: ConfModifyParams ): resp_lst = [] # 写回配置 for item in params.items: app, group, field = item.link conf = get_conf(app) conf[group][field] = item.newVal try: conf.commit() resp_lst.append(ModifyRespModel( id=item.link, errcode=0, errmsg='' )) except Exception as err: from traceback import print_exc, format_exc print_exc() resp_lst.append(ModifyRespModel( id=item.link, errcode=1, errmsg=format_exc() )) return ConfModifyResp(data=resp_lst)
def js_session(source, timeout=None, engine=None): from request.utils import jsruntime worker = get_conf('worker')['jsruntime'] timeout = timeout or worker.get('timeout', None) if not engine: engine = jscaller.engine.JSEngine( name=worker['name'], source=worker['source'], shell=worker['shell'], version=worker['version'], encoding='utf-8', ) if os.path.isfile(source): session = jscaller.session else: session = jscaller.Session with session(source, timeout, engine) as sess: yield sess req = jsruntime(sess) task = req.start_request() result = task.result() return result
def main(): get_client('script') for k, v in get_conf('worker').items(): register_worker(name=k, max_concurrent=v.get('max_concurrent', None), async_type=v['async'], independent=v.get('independent', False), ep=get_ep(v.get('entrypoint', 'requester'))) conf = get_conf('app').taskflow gateway = conf['gateway'] host = conf.get('host', None) port = conf.get('port', None) if not host: host = gateway.hostname port = gateway.port uvicorn.run(app, host=host, port=port) # TODO: 先正常暂停持久化任务后再进行关闭线程。 pool.shutdown()
def _request(self): return APIRequestMethod( parent=self, session=session, gateway=get_conf('app')[server_name]['gateway'].geturl(), path=r.path, methods=r.methods, doc=r.endpoint.__doc__, description=r.description, response_model=r.response_model, hooks=h)
def init_worker(): config = get_conf('worker') pls = {'script', 'function'} for pl in pls: register_worker( name=pl, max_concurrent=config[pl].get('max_concurrent', None), async_type=config[pl]['async'], independent=config[pl].get('independent', False), ep=get_ep(config[pl].get('entrypoint', 'requester')) )
def run_async(self) -> Any: """ 异步运行任务。""" conf = get_conf('taskflow') ctxmgr_value = { flow_mgr: self, glb['config']: conf['global'], glb['task']: self, } with ExitStack() as ctx_stack: for ctxmgr, value in ctxmgr_value.items(): ctx_stack.enter_context(ctxmgr.apply(value)) return self._task.start_request()
def apply_requester_context(request: Requester): """ """ var_value = { progress_mapping_context: request.progress, request_mapping_context: request, config_context: dict(get_conf('worker').get(request.NAME, {})), self_request: request, } with ExitStack() as ctx_stack: contexts = [ ctx_stack.enter_context(var.apply(value)) for var, value in var_value.items() ] yield
def main(): init_scripts() init_worker() conf = get_conf('app').script gateway = conf['gateway'] host = conf.get('host', None) port = conf.get('port', None) if not host: host = gateway.hostname port = gateway.port uvicorn.run(app, host=host, port=port) pool.shutdown()
def client_hook(self, params: dict, headers: dict, cookies: dict, *, name: str = None): # 改hook需要配合raw参数使用 start = time.time() try: result = yield params, headers, cookies except ClientResponseError as err: result = {'code': err.code, 'msg': err.msg} return AppRespModel(name=name, latency=(time.time() - start) * 1000, gateway=get_conf('app')[name].gateway.geturl(), **result)
async def _worker(index, layer): """ 执行流程。""" ctxmgr_value = { a: index, script['key']: f'{id(script_req):x}', script['config']: script_req.getdata('config', {}), script['basecnf']: dict(get_conf('script')['base']), # 方便获取脚本数据 script['__getitem__']: script_req.__getitem__, } with ExitStack() as stack: for ctxmgr, value in ctxmgr_value.items(): stack.enter_context(ctxmgr.apply(value)) stack.enter_context(layer) async with sema: return await layer.run()
async def get_state(params: SystemStateParams = Depends(SystemStateParams)): async def state(app_name): worker = get_worker('default') cli = get_client(app_name, timeout=params.timeout, hook=partial(client_hook, name=app_name), raw=True) result = await executor.submit(worker, (cli.get_state, )) return result results = await asyncio.gather( *[state(name) for name in ['script', 'taskflow']]) api_state = AppRespModel(name='api', latency=0, gateway=get_conf('app')['api'].gateway.geturl(), **dict(await _simple_state(params))) results = [api_state] + results return AppResponse(data=results)
def __init__( self, headers=None, cookies=None, timeout=None, proxies=None, verify=None, raw=False, hook=None, ): if timeout is None: timeout = get_conf('app')[self.__cli_name__].get('timeout', 10) self._timeout = timeout self._headers = headers self._cookies = cookies self._proxies = proxies self._verify = verify self._raw = raw self._hook = hook
async def query_conf( params: QueryConfigureParams = Depends(QueryConfigureParams) ): conf_field = [] for name in ['app', 'script', 'taskflow', 'worker']: conf = get_conf(name) conf_field.append({ 'title': name, 'name': name, 'groups': [{ 'title': k0, 'name': k0, 'items': [ section.get_field(k1) for k1 in section.keys() ] } for k0, section in conf.items()] }) return ConfQueryResp(data=[ AppConfModel(**conf) for conf in conf_field ])
def run_app(name, *args, **kwargs): conf = get_conf('app') app = conf[name] module = importlib.import_module(app.module) return getattr(module, app.entrypoint)(*args, **kwargs)
# TODO: 使用Sqlite来存储日志 from helper.conf import get_conf __logger_conf = get_conf('logger') def get_logger(name): __logger_conf[name] raise NotImplementedError()