def main():
conf = get_conf('app')
gateway = conf.api.gateway
default_conf = get_conf('worker')['default']
register_worker('default', default_conf['max_concurrent'],
default_conf['async'],
default_conf.get('independent', False),
get_ep(default_conf['entrypoint']))
uvicorn.run(app, host=gateway.hostname, port=gateway.port)
pool.shutdown()
async def modify_conf(
params: ConfModifyParams
):
resp_lst = []
# 写回配置
for item in params.items:
app, group, field = item.link
conf = get_conf(app)
conf[group][field] = item.newVal
try:
conf.commit()
resp_lst.append(ModifyRespModel(
id=item.link,
errcode=0,
errmsg=''
))
except Exception as err:
from traceback import print_exc, format_exc
print_exc()
resp_lst.append(ModifyRespModel(
id=item.link,
errcode=1,
errmsg=format_exc()
))
return ConfModifyResp(data=resp_lst)
def js_session(source, timeout=None, engine=None):
from request.utils import jsruntime
worker = get_conf('worker')['jsruntime']
timeout = timeout or worker.get('timeout', None)
if not engine:
engine = jscaller.engine.JSEngine(
name=worker['name'],
source=worker['source'],
shell=worker['shell'],
version=worker['version'],
encoding='utf-8',
)
if os.path.isfile(source):
session = jscaller.session
else:
session = jscaller.Session
with session(source, timeout, engine) as sess:
yield sess
req = jsruntime(sess)
task = req.start_request()
result = task.result()
return result
def main():
get_client('script')
for k, v in get_conf('worker').items():
register_worker(name=k,
max_concurrent=v.get('max_concurrent', None),
async_type=v['async'],
independent=v.get('independent', False),
ep=get_ep(v.get('entrypoint', 'requester')))
conf = get_conf('app').taskflow
gateway = conf['gateway']
host = conf.get('host', None)
port = conf.get('port', None)
if not host:
host = gateway.hostname
port = gateway.port
uvicorn.run(app, host=host, port=port)
# TODO: 先正常暂停持久化任务后再进行关闭线程。
pool.shutdown()
def _request(self):
return APIRequestMethod(
parent=self,
session=session,
gateway=get_conf('app')[server_name]['gateway'].geturl(),
path=r.path,
methods=r.methods,
doc=r.endpoint.__doc__,
description=r.description,
response_model=r.response_model,
hooks=h)
def init_worker():
config = get_conf('worker')
pls = {'script', 'function'}
for pl in pls:
register_worker(
name=pl,
max_concurrent=config[pl].get('max_concurrent', None),
async_type=config[pl]['async'],
independent=config[pl].get('independent', False),
ep=get_ep(config[pl].get('entrypoint', 'requester'))
)
def run_async(self) -> Any:
""" 异步运行任务。"""
conf = get_conf('taskflow')
ctxmgr_value = {
flow_mgr: self,
glb['config']: conf['global'],
glb['task']: self,
}
with ExitStack() as ctx_stack:
for ctxmgr, value in ctxmgr_value.items():
ctx_stack.enter_context(ctxmgr.apply(value))
return self._task.start_request()
def apply_requester_context(request: Requester):
""" """
var_value = {
progress_mapping_context: request.progress,
request_mapping_context: request,
config_context: dict(get_conf('worker').get(request.NAME, {})),
self_request: request,
}
with ExitStack() as ctx_stack:
contexts = [
ctx_stack.enter_context(var.apply(value))
for var, value in var_value.items()
]
yield
def main():
init_scripts()
init_worker()
conf = get_conf('app').script
gateway = conf['gateway']
host = conf.get('host', None)
port = conf.get('port', None)
if not host:
host = gateway.hostname
port = gateway.port
uvicorn.run(app, host=host, port=port)
pool.shutdown()
def client_hook(self,
params: dict,
headers: dict,
cookies: dict,
*,
name: str = None):
# 改hook需要配合raw参数使用
start = time.time()
try:
result = yield params, headers, cookies
except ClientResponseError as err:
result = {'code': err.code, 'msg': err.msg}
return AppRespModel(name=name,
latency=(time.time() - start) * 1000,
gateway=get_conf('app')[name].gateway.geturl(),
**result)
async def _worker(index, layer):
""" 执行流程。"""
ctxmgr_value = {
a: index,
script['key']: f'{id(script_req):x}',
script['config']: script_req.getdata('config', {}),
script['basecnf']: dict(get_conf('script')['base']),
# 方便获取脚本数据
script['__getitem__']: script_req.__getitem__,
}
with ExitStack() as stack:
for ctxmgr, value in ctxmgr_value.items():
stack.enter_context(ctxmgr.apply(value))
stack.enter_context(layer)
async with sema:
return await layer.run()
async def get_state(params: SystemStateParams = Depends(SystemStateParams)):
async def state(app_name):
worker = get_worker('default')
cli = get_client(app_name,
timeout=params.timeout,
hook=partial(client_hook, name=app_name),
raw=True)
result = await executor.submit(worker, (cli.get_state, ))
return result
results = await asyncio.gather(
*[state(name) for name in ['script', 'taskflow']])
api_state = AppRespModel(name='api',
latency=0,
gateway=get_conf('app')['api'].gateway.geturl(),
**dict(await _simple_state(params)))
results = [api_state] + results
return AppResponse(data=results)
def __init__(
self,
headers=None,
cookies=None,
timeout=None,
proxies=None,
verify=None,
raw=False,
hook=None,
):
if timeout is None:
timeout = get_conf('app')[self.__cli_name__].get('timeout', 10)
self._timeout = timeout
self._headers = headers
self._cookies = cookies
self._proxies = proxies
self._verify = verify
self._raw = raw
self._hook = hook
async def query_conf(
params: QueryConfigureParams = Depends(QueryConfigureParams)
):
conf_field = []
for name in ['app', 'script', 'taskflow', 'worker']:
conf = get_conf(name)
conf_field.append({
'title': name,
'name': name,
'groups': [{
'title': k0,
'name': k0,
'items': [
section.get_field(k1)
for k1 in section.keys()
]
} for k0, section in conf.items()]
})
return ConfQueryResp(data=[
AppConfModel(**conf)
for conf in conf_field
])
def run_app(name, *args, **kwargs):
conf = get_conf('app')
app = conf[name]
module = importlib.import_module(app.module)
return getattr(module, app.entrypoint)(*args, **kwargs)
# TODO: 使用Sqlite来存储日志
from helper.conf import get_conf
__logger_conf = get_conf('logger')
def get_logger(name):
__logger_conf[name]
raise NotImplementedError()