def load_component(self, layout, backend='ini', **setting):
"""加载组件管理器
可以进行多次加载
:parameter:
- `layout`: 当前调用的层次，web, service, task
- `backend`: 配置方式,目前支持ini
- `setting`: 该格式需要的设置参数
"""
layout = layout.lower()
configer = ConfigurationParser(backend, **setting)
# 加载需要管理连接池的组件
recorder('INFO', 'load connection component start')
with timing('ms', 10) as t:
if layout in ['service']:
fastweb.manager.SyncConnManager.setup(configer)
elif layout in ['web']:
fastweb.manager.AsynConnManager.configer = configer
ioloop.IOLoop.current().run_sync(fastweb.manager.AsynConnManager.setup)
recorder('INFO', 'load connection component successful -- {time}'.format(time=t))
# 加载不需要管理连接池的组件
recorder('INFO', 'load component start')
with timing('ms', 10) as t:
fastweb.manager.Manager.setup(layout, configer)
recorder('INFO', 'load component successful -- {time}'.format(time=t))
self.component_configers.append(configer)
return configer
def test():
conn = yield tornado_mysql.connect(**{'host': 'localhost', 'port': 3306, 'user': '******', 'password': '******'})
with timing('ms', 10) as ti:
for _ in range(30):
cur = conn.cursor(tornado_mysql.cursors.DictCursor)
with timing('ms', 10) as t:
yield cur.execute('SELECT * FROM mysql.user;')
print(t)
print('total:')
print(ti)
def query(self,
command,
value=1,
check=True,
allowable_errors=None,
**kwargs):
"""命令行操作
pymongo内部没有实现重试,需要进行重试
TODO:对不同的异常做不同的操作"""
if not self._db:
self.recorder('CRITICAL', 'please select db first!')
shell_command = 'db.runCommand(\n{cmd}\n)'.format(
cmd=dumps(command, indent=4, whole=4))
self.recorder(
'INFO', '{obj} command start\n{cmd}'.format(obj=self,
cmd=shell_command))
try:
with tool.timing('s', 10) as t:
response = self._db.command(command=command,
value=value,
check=check,
allowable_errors=allowable_errors,
**kwargs)
except pymongo.errors.PyMongoError as e:
self.recorder(
'ERROR', '{obj} command error [{msg}]'.format(obj=self, msg=e))
raise MongoError
self.recorder(
'INFO', '{obj} command successful\n{cmd} -- {time}'.format(
obj=self, cmd=shell_command, time=t))
self._response = self._parse_response(response)
return self._response
def call(self, *args, **kwargs):
"""同步调用"""
self.requestid = self.owner.requestid + '-' + self.name if self.owner else None
# TODO:多个同步任务的链式调用
with timing('ms', 10) as t:
self.recorder(
'INFO',
'synchronize call {obj} start\nArgument: {args} {kwargs}'.
format(obj=self, args=args, kwargs=kwargs))
# 调用task层时，调用方的requestid会成为本次任务的taskid， 任务的requesid也为透传id
result = yield torcelery.sync(self,
self.timeout,
task_id=self.requestid,
queue=self.queue,
exchange=self.exchange,
routing_key=self.routing_key,
*args,
**kwargs)
if not result:
self.recorder(
'ERROR',
'synchronize call {obj} timeout -- {t}'.format(obj=self,
ret=result,
t=t))
raise TaskError
self.recorder(
'INFO', 'synchronize call {obj} successful\n'
'Return:{ret} -- {t}'.format(obj=self, ret=result, t=t))
raise Return(result)
def call(self, *args, **kwargs):
"""同步调用"""
# TODO:多个同步任务的链式调用
self.requestid = self.owner.requestid + '-' + self.name if self.owner else None
with timing('ms', 10) as t:
self.recorder('INFO',
'synchronize call {task} start'.format(task=self))
# 调用task层时，调用方的requestid会成为本次任务的taskid， 任务的requesid也为透传id
result = self.apply_async(task_id=self.requestid,
queue=self.queue,
exchange=self.exchange,
routing_key=self.routing_key,
*args,
**kwargs)
while True:
if result.ready():
break
if not result:
self.recorder(
'ERROR',
'synchronize call {obj} timeout -- {t}'.format(obj=self,
ret=result,
t=t))
raise TaskError
self.recorder(
'INFO', 'synchronize call {obj} successful -- {ret} -- {t}'.format(
obj=self, ret=result, t=t))
return result.result
def _soap_request(self, retry, request):
if hasattr(self, 'requestid'):
_recorder = self.recorder
else:
_recorder = recorder
_recorder('INFO',
'soap request start {request}'.format(request=request))
with timing('ms', 10) as t:
try:
transport = Transport(operation_timeout=request.timeout)
client = CachingClient(wsdl=request.wsdl,
wsse=request.wsse,
transport=transport)
response = getattr(client.service,
request.function)(**request.kwargs)
except Exception as e:
# wried error, hold all exception
_recorder('ERROR', 'soap request error ({e}))'.format(e=e))
raise retry
_recorder(
'INFO', 'soap request successful\n{response} -- {time}'.format(
response=response, time=t))
return response
def query(self, command):
"""执行redis命令"""
try:
self._parse_command(command)
cmd = shlex.split(command)
self.recorder(
'INFO',
'{obj} query start\nCommand: {cmd}'.format(obj=self,
cmd=command))
with tool.timing('s', 10) as t:
if not self._client.is_connected():
yield self.connect()
response = yield self._client.call(*cmd)
response = self._parse_response(response)
self.recorder(
'INFO',
'{obj} query success\nCommand: {cmd}\nResponse: {res} -- {time}'
.format(obj=self,
cmd=py.utf8(command),
res=py.utf8(response),
time=t))
except torConnectionError as e:
self.recorder(
'ERROR', '{obj} connection error [{msg}]'.format(obj=self,
msg=e))
raise RedisError
raise Return(response)
def query(self, command):
"""命令行操作
ConnectionError可能是超出连接最大数
TimeoutError可能是连接不通"""
try:
self._parse_command(command)
cmd = shlex.split(command)
self.recorder(
'INFO', '{obj} query start\n{cmd}'.format(obj=self,
cmd=command))
with tool.timing('s', 10) as t:
response = self._client.execute_command(*cmd)
response = self._parse_response(response)
self.recorder(
'INFO', '{obj} query successful\n{cmd} -- {time}'.format(
obj=self, cmd=py.utf8(command), time=t))
except (ConnectionError, TimeoutError) as e:
# redis内部对这两种异常进行了重试操作
self.recorder(
'ERROR', '{obj} connection error [{msg}]'.format(obj=self,
msg=e))
raise RedisError
except ResponseError as e:
self.recorder('ERROR', '{obj} query error [{msg}]'.format(obj=self,
msg=e))
raise RedisError
return response
def run(self, *args, **kwargs):
"""任务处理
转发给具体执行对象的run方法"""
# Components中生成requestid
self._worker_obj = type('Worker',
(self._task_cls, SyncComponents, IFaceWorker),
{'request': self.request})()
self._worker_obj.setting = self._setting
self._worker_obj.requestid = self.request.id
if hasattr(self._worker_obj, 'run'):
self._worker_obj.recorder(
'IMPORTANT',
'{obj} start\nRequest:\n{request}\nArgument: {args}\t{kwargs}'.
format(obj=self,
request=json.dumps(self.request.as_execution_options(),
indent=4),
args=args,
kwargs=kwargs))
with timing('ms', 10) as t:
ret = self._worker_obj.run(*args, **kwargs)
self._worker_obj.recorder(
'IMPORTANT', '{obj} end\nReturn: {r} -- {t}'.format(obj=self,
r=ret,
t=t))
return ret
else:
self._worker_obj.recorder(
'CRITICAL', '{obj} must have run function!'.format(obj=self))
raise TaskError
def after_return(self, status, retval, task_id, args, kwargs, einfo):
"""任务返回回调函数转发
Arguments:
status (str): Current task state.
retval (Any): Task return value/exception.
task_id (str): Unique id of the task.
args (Tuple): Original arguments for the task.
kwargs (Dict): Original keyword arguments for the task.
einfo (~billiard.einfo.ExceptionInfo): Exception information.
Returns:
None: The return value of this handler is ignored.
"""
if hasattr(self._worker_obj, 'after_return'):
self._worker_obj.recorder(
'DEBUG', '{obj} after return callback start'.format(obj=self))
with timing('ms', 10) as t:
self._worker_obj.after_return(status, retval, task_id, args,
kwargs, einfo)
self._worker_obj.recorder(
'DEBUG',
'{obj} after return callback end -- {t}'.format(obj=self, t=t))
self._worker_obj.release()
def on_retry(self, exc, task_id, args, kwargs, einfo):
"""任务重试回调函数转发
This is run by the worker when the task is to be retried.
Arguments:
exc (Exception): The exception sent to :meth:`retry`.
task_id (str): Unique id of the retried task.
args (Tuple): Original arguments for the retried task.
kwargs (Dict): Original keyword arguments for the retried task.
einfo (~billiard.einfo.ExceptionInfo): Exception information.
Returns:
None: The return value of this handler is ignored.
"""
if hasattr(self._worker_obj, 'on_retry'):
self._worker_obj.recorder(
'INFO', '{obj} retry callback start'.format(obj=self))
with timing('ms', 10) as t:
self._worker_obj.on_retry(exc, task_id, args, kwargs, einfo)
self._worker_obj.recorder(
'INFO', '{obj} retry callback end -- {t}'.format(obj=self,
t=t))
else:
self._worker_obj.recorder(
'ERROR',
'{obj} retry\nTaskid: {id}\nException: {e}'.format(obj=self,
id=task_id,
e=exc))
def on_failure(self, exc, task_id, args, kwargs, einfo):
"""
任务失败回调函数转发
This is run by the worker when the task fails.
Arguments:
exc (Exception): The exception raised by the task.
task_id (str): Unique id of the failed task.
args (Tuple): Original arguments for the task that failed.
kwargs (Dict): Original keyword arguments for the task that failed.
einfo (~billiard.einfo.ExceptionInfo): Exception information.
Returns:
None: The return value of this handler is ignored.
"""
self._worker_obj.recorder(
'ERROR',
'{obj} failure\nTaskid: {id}\nException: {e}'.format(obj=self,
id=task_id,
e=exc))
if hasattr(self._worker_obj, 'on_failure'):
self._worker_obj.recorder(
'INFO', '{obj} failure callback start'.format(obj=self))
with timing('ms', 10) as t:
r = self._worker_obj.on_failure(exc, task_id, args, kwargs,
einfo)
self._worker_obj.recorder(
'INFO', '{obj} failure callback end -- {t}'.format(obj=self,
t=t))
return r
def on_success(self, retval, task_id, args, kwargs):
"""
任务成功回调函数转发
Run by the worker if the task executes successfully.
Arguments:
retval (Any): The return value of the task.
task_id (str): Unique id of the executed task.
args (Tuple): Original arguments for the executed task.
kwargs (Dict): Original keyword arguments for the executed task.
Returns:
None: The return value of this handler is ignored.
"""
if hasattr(self._worker_obj, 'on_success'):
self._worker_obj.recorder(
'INFO', '{obj} success callback start'.format(obj=self))
with timing('ms', 10) as t:
r = self._worker_obj.on_success(retval, task_id, args, kwargs)
self._worker_obj.recorder(
'INFO', '{obj} success callback end -- {t}'.format(obj=self,
t=t))
return r
def _http_request(self, retry, request):
"""http请求
:parameter:
- `request`:http请求
"""
if hasattr(self, 'requestid'):
_recorder = self.recorder
else:
_recorder = recorder
_recorder('INFO',
'http request start {request}'.format(request=request))
with timing('ms', 10) as t:
try:
response = HttpClient().fetch(request)
except RequestHTTPError as ex:
_recorder(
'ERROR',
'http request error {request} {e}'.format(request=request,
e=ex))
raise retry
_recorder(
'INFO', 'http request successful\n{response} -- {time}'.format(
response=response.status_code, time=t))
return response
def call_subprocess(self, command, stdin_data=None, stdin_async=True):
"""命令行调用
:parameter:
- `command`:命令行
- `stdin_data`:传入数据
"""
# TODO:待优化
self.recorder('INFO',
'call subprocess start <{cmd}>'.format(cmd=command))
with timing('ms', 10) as t:
stdin = Subprocess.STREAM if stdin_async else subprocess.PIPE
sub_process = Subprocess(shlex.split(command),
stdin=stdin,
stdout=Subprocess.STREAM,
stderr=Subprocess.STREAM)
try:
if stdin_data:
if stdin_async:
yield Task(sub_process.stdin.write, stdin_data)
else:
sub_process.stdin.write(stdin_data)
if stdin_async or stdin_data:
sub_process.stdin.close()
result, error = yield [
Task(sub_process.stdout.read_until_close),
Task(sub_process.stderr.read_until_close)
]
except (OSError, ValueError) as ex:
self.recorder(
'ERROR',
'call subprocess <{cmd} <{time}> {e} {msg}'.format(
cmd=command,
time=t,
e=ex,
msg=result.strip() if result else error.strip()))
raise SubProcessError
if sub_process.returncode:
self.recorder(
'ERROR', 'call subprocess <{cmd}> <{time}> {msg}'.format(
cmd=command,
time=t,
msg=result.strip() if result else error.strip()))
raise SubProcessError
self.recorder(
'INFO', 'call subprocess <{cmd}> <{time} {msg}>'.format(
cmd=command,
time=t,
msg=result.strip() if result else error.strip()))
raise Return((result, error))
def call_subprocess(self, command, stdin_data=None, timeout=None):
"""命令行调用
:parameter:
- `command`:命令行
- `stdin_data`:传入数据
"""
if hasattr(self, 'requestid'):
_recorder = self.recorder
else:
_recorder = recorder
_recorder('INFO', 'call subprocess start\n{cmd}'.format(cmd=command))
with timing('ms', 10) as t:
sub_process = subprocess.Popen(shlex.split(command),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
result, error = sub_process.communicate(stdin_data,
timeout=timeout)
except (OSError, ValueError) as ex:
_recorder(
'ERROR', 'call subprocess\n({cmd}) ({e}) '.format(
cmd=command,
e=ex,
msg=result.strip() if result else error.strip()))
raise SubProcessError
except subprocess.TimeoutExpired:
_recorder(
'ERROR', 'call subprocess timeout [{timeout}]'.format(
timeout=timeout))
raise SubProcessTimeoutError
finally:
sub_process.kill()
if sub_process.returncode != 0:
_recorder(
'ERROR',
'call subprocess error ({cmd}) <{time}> {msg}'.format(
cmd=command,
time=t,
msg=result.strip() if result else error.strip()))
raise SubProcessError
_recorder(
'INFO',
'call subprocess successful\n{cmd}\n{msg}\n<{time}>'.format(
cmd=command,
time=t,
msg=result.strip() if result else error.strip()))
return result, error
def _query(self, sql, retry, args):
if not self._cur:
self._cur = self._conn.cursor(pymysql.cursors.DictCursor)
elif self._cur and not self._event:
self._cur.close()
self._cur = self._conn.cursor(pymysql.cursors.DictCursor)
try:
self._format_sql(sql, args)
self.recorder(
'INFO', '{obj} query start ({threadid})\n{sql}'.format(
obj=self,
threadid=self._conn.server_thread_id[0],
sql=self._sql))
with tool.timing('s', 10) as t:
self._cur.execute(sql, args)
self.recorder(
'INFO',
'{obj} query successful\n{sql}\t[{time}]\t[{effect}]'.format(
obj=self, sql=self._sql, time=t,
effect=self._cur.rowcount))
except pymysql.OperationalError as e:
self.recorder(
'ERROR', '{obj} mysql has gone away [{msg}]'.format(obj=self,
msg=e))
self.reconnect()
raise retry
except (pymysql.IntegrityError, pymysql.ProgrammingError) as e:
self.recorder(
'ERROR',
'{obj} query error\n{sql}\n[{msg}]'.format(obj=self,
sql=self._sql,
msg=e))
raise MysqlError
except (KeyError, TypeError) as e:
self.recorder(
'ERROR',
'{obj} sql format error\n{sql}\n{args}\n[{msg}]'.format(
obj=self, sql=sql, args=args, msg=e))
raise MysqlError
if not self._event:
self.commit()
effect = self._cur.rowcount
self._cur.close()
return effect
def anonymous(p, seq, ipo, opo):
oproc = getattr(module,
'Processor')(handler=self._handlers())
oproc._handler.requestid = seq if len(
str(seq)) > 8 else oproc._handler.requestid
oproc._handler.recorder(
'IMPORTANT',
'{obj}\nremote call [{name}]'.format(obj=self,
name=_func_name))
with timing('ms', 8) as t:
_func(oproc, seq, ipo, opo)
oproc._handler.release()
oproc._handler.recorder(
'IMPORTANT',
'{obj}\nremote call [{name}] success -- {t}'.format(
obj=self, name=_func_name, t=t))
def call_async(self, *args, **kwargs):
"""异步调用"""
self.requestid = self.owner.requestid + '-' + self.name if self.owner else None
with timing('ms', 10) as t:
self.recorder('INFO',
'asynchronous call {obj} start'.format(obj=self))
# 调用task层时，调用方的requestid会成为本次任务的taskid， 任务的requesid也为透传id
taskid = self.apply_async(task_id=self.requestid,
queue=self.queue,
exchange=self.exchange,
routing_key=self.routing_key,
*args,
**kwargs)
self.recorder(
'INFO',
'asynchronous call {obj} successful -- {t}'.format(obj=self, t=t))
return taskid
def call_async(self, *args, **kwargs):
"""异步调用"""
self.requestid = self.owner.requestid + '-' + self.name if self.owner else None
with timing('ms', 10) as t:
self.recorder(
'INFO',
'asynchronous call {obj} start\nArgument: {args} {kwargs}'.
format(obj=self, args=args, kwargs=kwargs))
# 调用task层时，调用方的requestid会成为本次任务的taskid， 任务的requesid也为透传id
taskid = yield torcelery. async (self,
task_id=self.requestid,
queue=self.queue,
exchange=self.exchange,
routing_key=self.routing_key,
*args,
**kwargs)
self.recorder(
'INFO',
'asynchronous call {obj} successful -- {t}'.format(obj=self, t=t))
raise Return(taskid)
def _http_request(self, retry, request):
"""http请求
:parameter:
- `request`:http请求
"""
self.recorder('INFO',
'http request start\n{request}'.format(request=request))
with timing('ms', 10) as t:
try:
response = yield AsyncHTTPClient().fetch(request)
except HTTPError as ex:
self.recorder(
'ERROR', 'http request error {request} ({e})'.format(
request=request, e=ex))
raise retry
self.recorder(
'INFO', 'http request successful\t({response} <{time}>)'.format(
response=response.code, time=t))
raise Return(response)
def call_subprocess(obj):
returncode = subprocess.call(shlex.split('ls'))
print(('returncode: {}'.format(returncode)))
def call_subprocess_gevent():
returncode = gevent.subprocess.call(shlex.split('ls'))
print(returncode)
if __name__ == '__main__':
b = True
x = 1000
with timing('s', 8) as t:
if b:
for i in range(x):
FThread("Thread-{}".format(i), call_subprocess,
frequency=1).start()
FThread.stop()
else:
pool = Pool(x)
for i in range(x):
pool.spawn(call_subprocess_gevent)
pool.join()
print(('total cost: {}'.format(t)))