class Engine(BaseEngine, Signaler):
#
# 引擎本身只需实现 BaseEngine 接口，信号处理属于额外增强功能。
# 子进程创建、退出都属于系统信号范畴，统一由 Singaler 处理。
#
def __init__(self, server):
self._server = server
self._pool = ThreadPool(CPUS * 4)
self._listen_sock = None
self._wsgi_server = None
BaseEngine.__init__(self, server)
Signaler.__init__(self)
def run(self):
from engine.config import HOST, PORT
self._listen_sock = socket(family=AF_INET)
self._listen_sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self._listen_sock.bind((HOST, PORT))
self._listen_sock.listen(2048)
self._listen_sock.setblocking(0)
self.fork_workers(WORKERS or CPUS + 1)
self.parent_execute()
def fork_workers(self, num):
for i in range(num):
if fork() == 0:
self.worker_execute()
exit(0)
def worker_execute(self):
Signaler.worker_execute(self)
# 启动服务器。
kwargs = HTTPS and \
{k: app_path("ssl/" + v) for k, v in (("keyfile", HTTPS_KEY), ("certfile", HTTPS_CERT))} or \
{}
self._wsgi_server = WSGIServer(self._listen_sock, self._server.execute, log=None, **kwargs)
self._wsgi_server.serve_forever()
# 等待所有处理结束，超时 10 秒。
hasattr(self._wsgi_server, "__graceful__") and gwait(timeout=10)
def worker_stop(self, graceful):
stop = lambda *args: self._wsgi_server and self._wsgi_server.stop()
graceful and (setattr(self._wsgi_server, "__graceful__", True), stop()) or stop()
def async_execute(self, func, *args, **kwargs):
e = Event()
g = self._pool.apply_async(func, args, kwargs, callback=lambda ret: e.set())
e.wait()
return g.get()
def invalid():
tp = ThreadPool(10)
# It's invalid to use callback.
greenlets = [
tp.apply_async(foo, args=(i, ), kwds=None, callback=callback_)
for i in xrange(1, 300)
]
gevent.joinall(greenlets)
class TestPool(TestCase):
__timeout__ = 5
size = 1
def setUp(self):
greentest.TestCase.setUp(self)
self.pool = ThreadPool(self.size)
def test_apply(self):
papply = self.pool.apply
self.assertEqual(papply(sqr, (5,)), sqr(5))
self.assertEqual(papply(sqr, (), {'x': 3}), sqr(x=3))
def test_map(self):
pmap = self.pool.map
self.assertEqual(pmap(sqr, range(10)), list(map(sqr, range(10))))
self.assertEqual(pmap(sqr, range(100)), list(map(sqr, range(100))))
def test_async(self):
res = self.pool.apply_async(sqr, (7, TIMEOUT1,))
get = TimingWrapper(res.get)
self.assertEqual(get(), 49)
self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)
def test_async_callback(self):
result = []
res = self.pool.apply_async(sqr, (7, TIMEOUT1,), callback=lambda x: result.append(x))
get = TimingWrapper(res.get)
self.assertEqual(get(), 49)
self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)
gevent.sleep(0) # let's the callback run
assert result == [49], result
def test_async_timeout(self):
res = self.pool.apply_async(sqr, (6, TIMEOUT2 + 0.2))
get = TimingWrapper(res.get)
self.assertRaises(gevent.Timeout, get, timeout=TIMEOUT2)
self.assertAlmostEqual(get.elapsed, TIMEOUT2, 1)
self.pool.join()
def test_imap(self):
it = self.pool.imap(sqr, range(10))
self.assertEqual(list(it), list(map(sqr, range(10))))
it = self.pool.imap(sqr, range(10))
for i in range(10):
self.assertEqual(six.advance_iterator(it), i * i)
self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
it = self.pool.imap(sqr, range(1000))
for i in range(1000):
self.assertEqual(six.advance_iterator(it), i * i)
self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
def test_imap_random(self):
it = self.pool.imap(sqr_random_sleep, range(10))
self.assertEqual(list(it), list(map(sqr, range(10))))
def test_imap_unordered(self):
it = self.pool.imap_unordered(sqr, range(1000))
self.assertEqual(sorted(it), list(map(sqr, range(1000))))
it = self.pool.imap_unordered(sqr, range(1000))
self.assertEqual(sorted(it), list(map(sqr, range(1000))))
def test_imap_unordered_random(self):
it = self.pool.imap_unordered(sqr_random_sleep, range(10))
self.assertEqual(sorted(it), list(map(sqr, range(10))))
def test_terminate(self):
result = self.pool.map_async(sleep, [0.1] * ((self.size or 10) * 2))
gevent.sleep(0.1)
kill = TimingWrapper(self.pool.kill)
kill()
assert kill.elapsed < 0.5, kill.elapsed
result.join()
def sleep(self, x):
sleep(float(x) / 10.)
return str(x)
def test_imap_unordered_sleep(self):
# testing that imap_unordered returns items in competion order
result = list(self.pool.imap_unordered(self.sleep, [10, 1, 2]))
if self.pool.size == 1:
expected = ['10', '1', '2']
else:
expected = ['1', '2', '10']
self.assertEqual(result, expected)
class TestPool(TestCase):
__timeout__ = 5
size = 1
def setUp(self):
greentest.TestCase.setUp(self)
self.pool = ThreadPool(self.size)
def test_apply(self):
papply = self.pool.apply
self.assertEqual(papply(sqr, (5, )), sqr(5))
self.assertEqual(papply(sqr, (), {'x': 3}), sqr(x=3))
def test_map(self):
pmap = self.pool.map
self.assertEqual(pmap(sqr, range(10)), list(map(sqr, range(10))))
self.assertEqual(pmap(sqr, range(100)), list(map(sqr, range(100))))
def test_async(self):
res = self.pool.apply_async(sqr, (
7,
TIMEOUT1,
))
get = TimingWrapper(res.get)
self.assertEqual(get(), 49)
self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)
def test_async_callback(self):
result = []
res = self.pool.apply_async(sqr, (
7,
TIMEOUT1,
),
callback=lambda x: result.append(x))
get = TimingWrapper(res.get)
self.assertEqual(get(), 49)
self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)
gevent.sleep(0) # let's the callback run
assert result == [49], result
def test_async_timeout(self):
res = self.pool.apply_async(sqr, (6, TIMEOUT2 + 0.2))
get = TimingWrapper(res.get)
self.assertRaises(gevent.Timeout, get, timeout=TIMEOUT2)
self.assertAlmostEqual(get.elapsed, TIMEOUT2, 1)
self.pool.join()
def test_imap(self):
it = self.pool.imap(sqr, range(10))
self.assertEqual(list(it), list(map(sqr, range(10))))
it = self.pool.imap(sqr, range(10))
for i in range(10):
self.assertEqual(six.advance_iterator(it), i * i)
self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
it = self.pool.imap(sqr, range(1000))
for i in range(1000):
self.assertEqual(six.advance_iterator(it), i * i)
self.assertRaises(StopIteration, lambda: six.advance_iterator(it))
def test_imap_random(self):
it = self.pool.imap(sqr_random_sleep, range(10))
self.assertEqual(list(it), list(map(sqr, range(10))))
def test_imap_unordered(self):
it = self.pool.imap_unordered(sqr, range(1000))
self.assertEqual(sorted(it), list(map(sqr, range(1000))))
it = self.pool.imap_unordered(sqr, range(1000))
self.assertEqual(sorted(it), list(map(sqr, range(1000))))
def test_imap_unordered_random(self):
it = self.pool.imap_unordered(sqr_random_sleep, range(10))
self.assertEqual(sorted(it), list(map(sqr, range(10))))
def test_terminate(self):
result = self.pool.map_async(sleep, [0.1] * ((self.size or 10) * 2))
gevent.sleep(0.1)
kill = TimingWrapper(self.pool.kill)
kill()
assert kill.elapsed < 0.5, kill.elapsed
result.join()
def sleep(self, x):
sleep(float(x) / 10.)
return str(x)
def test_imap_unordered_sleep(self):
# testing that imap_unordered returns items in competion order
result = list(self.pool.imap_unordered(self.sleep, [10, 1, 2]))
if self.pool.size == 1:
expected = ['10', '1', '2']
else:
expected = ['1', '2', '10']
self.assertEqual(result, expected)
class Engine(BaseEngine, Signaler):
#
# 引擎本身只需实现 BaseEngine 接口，信号处理属于额外增强功能。
# 子进程创建、退出都属于系统信号范畴，统一由 Singaler 处理。
#
def __init__(self, server):
self._server = server
self._pool = ThreadPool(CPUS * 4)
self._listen_sock = None
self._wsgi_server = None
BaseEngine.__init__(self, server)
Signaler.__init__(self)
def run(self):
self._listen_sock = socket(family=AF_INET)
self._listen_sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self._listen_sock.bind((HOST, PORT))
self._listen_sock.listen(2048)
self._listen_sock.setblocking(0)
self.fork_workers(WORKERS or CPUS + 1)
self.parent_execute()
def fork_workers(self, num):
for i in range(num):
if fork() == 0:
self.worker_execute()
exit(0)
def worker_execute(self):
Signaler.worker_execute(self)
# 启动服务器。
kwargs = HTTPS and \
{k: app_path("ssl/" + v) for k, v in (("keyfile", HTTPS_KEY), ("certfile", HTTPS_CERT))} or \
{}
self._wsgi_server = WSGIServer(self._listen_sock,
self._server.execute,
log=None,
**kwargs)
self._wsgi_server.serve_forever()
# 等待所有处理结束，超时 10 秒。
hasattr(self._wsgi_server, "__graceful__") and gwait(timeout=10)
def worker_stop(self, graceful):
stop = lambda *args: self._wsgi_server and self._wsgi_server.stop()
graceful and (setattr(self._wsgi_server, "__graceful__",
True), stop()) or stop()
def async_execute(self, func, *args, **kwargs):
e = Event()
g = self._pool.apply_async(func,
args,
kwargs,
callback=lambda ret: e.set())
e.wait()
return g.get()
class Engine(BaseEngine, Signaler):
#
# 引擎本身只需实现 BaseEngine 接口，信号处理属于额外增强功能。
# 子进程创建、退出都属于系统信号范畴，统一由 Singaler 处理。
#
def __init__(self, server):
self._server = server
self._pool = ThreadPool(settings.CPUS * 4)
self._listen_sock = None
self._wsgi_server = None
BaseEngine.__init__(self, server)
Signaler.__init__(self)
def run(self):
if settings.NEED_PATCH_SOCKET_SSL:
# gevent.pywsgi启动
from gevent.monkey import patch_socket, patch_ssl
patch_socket()
# 在patch socket之后，如果使用https会出错，需要连ssl也patch掉
patch_ssl()
if not settings.NEED_GEVENT_THREADPOOL:
def sync(func, *args, **kwargs):
return func(*args, **kwargs)
self.async_execute = sync
self._listen_sock = socket(family=AF_INET)
self._listen_sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self._listen_sock.bind((settings.HOST, settings.PORT))
self._listen_sock.listen(2048)
self._listen_sock.setblocking(0)
if settings.WORKERS == 1:
# 只有一个worker 启用单进程运行
self.serve_forever()
else:
self.fork_workers(settings.WORKERS or settings.CPUS + 1)
self.parent_execute()
def fork_workers(self, num):
for i in range(num):
if fork() == 0:
self.worker_execute()
exit(0)
def worker_execute(self):
Signaler.worker_execute(self)
self.serve_forever()
# 等待所有处理结束，超时 10 秒。
hasattr(self._wsgi_server, "__graceful__") and gwait(timeout=10)
def serve_forever(self):
# 启动服务器。
kwargs = settings.HTTPS and \
{k: app_path("ssl/" + v) for k, v in (("keyfile", settings.HTTPS_KEY), ("certfile", settings.HTTPS_CERT))} or \
{}
if settings.SUPPORT_WEBSOCKET:
from geventwebsocket.handler import WebSocketHandler
kwargs["handler_class"] = WebSocketHandler
self._wsgi_server = WSGIServer(self._listen_sock, self._server.execute, log=None, **kwargs)
self._wsgi_server.serve_forever()
def worker_stop(self, graceful):
stop = lambda *args: self._wsgi_server and self._wsgi_server.stop()
graceful and (setattr(self._wsgi_server, "__graceful__", True), stop()) or stop()
def async_execute(self, func, *args, **kwargs):
e = Event()
g = self._pool.apply_async(func, args, kwargs, callback=lambda ret: e.set())
e.wait()
return g.get()
def invalid1():
tp = ThreadPool(10)
for i in xrange(1, 300):
# It's invalid to use callback.
tp.apply_async(foo, args=(i, ), kwds=None, callback=callback_)
tp.join()
