def _run(self):
"""Main loop; reap and process pkts"""
try:
args = self.orbname, self.select, self.reject
print repr(self.orbname)
print repr(args)
with OrbreapThr(*args, timeout=1, queuesize=8, after=self.tafter) as orbreapthr:
log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
self.reject))
self.timeoff = self.timeon = datetime.utcnow()
spawn(self._status_printer).link_exception(self._janitor)
threadpool = ThreadPool(maxsize=1)
try:
while True:
try:
success, value = threadpool.spawn(
wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
timestamp = datetime.utcnow()
if not success:
raise value
except (Timeout, NoData), e:
log.debug("orbreapthr.get exception %r" % type(e))
pass
else:
if value is None:
raise Exception('Nothing to publish')
self._process(value, timestamp)
finally:
# This blocks until all threads in the pool return. That's
# critical; if the orbreapthr dies before the get thread,
# segfaults ensue.
threadpool.kill()
except Exception, e:
log.error("OrbPktSrc terminating due to exception", exc_info=True)
raise
class 协程池():
"""
创建一个协程池运行任务
pool = 协程池(协程数量=2)
任务列表 = []
for i in range(10):
任务列表.append(pool.创建任务(工作线程4,i))
pool.运行任务(任务列表)
"""
def __init__(self, 协程数量):
self.pool = ThreadPool(协程数量)
def 等待协程完成任务(self, 任务列表):
gevent.joinall(任务列表)
def 投递任务(self, 任务函数, *args, **kwargs):
return self.pool.spawn(任务函数, *args, **kwargs)
def 等待(self):
"""等待所有任务完成"""
self.pool.join()
# gevent.wait()
def 关闭(self):
"""等待所有任务完成"""
self.pool.kill()
class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
"""
A version of :class:`concurrent.futures.ThreadPoolExecutor` that
always uses native threads, even when threading is monkey-patched.
TODO: Gevent has also implemented [ThreadPoolExecutor](https://github.com/gevent/gevent/blob/master/src/gevent/threadpool.py#L454)
to be released in next version 1.2.0. We should move to that implementation when it is released.
"""
def __init__(self, max_workers):
super(ThreadPoolExecutor, self).__init__(max_workers)
self._threadpool = ThreadPool(max_workers)
def submit(self, fn, *args, **kwargs):
future = super(ThreadPoolExecutor, self).submit(fn, *args, **kwargs)
with self._shutdown_lock:
work_item = self._work_queue.get()
assert work_item.fn is fn
self._threadpool.spawn(work_item.run)
return future
def shutdown(self, wait=True):
super(ThreadPoolExecutor, self).shutdown(wait)
self._threadpool.kill()
kill = shutdown # greentest compat
def _adjust_thread_count(self):
# Does nothing. We don't want to spawn any "threads",
# let the threadpool handle that.
pass
def _run(self):
try:
args = self.orbname, self.select, self.reject
# TODO Review this queue size
# TODO Review reasoning behind using OrbreapThr vs. normal ORB API
# I think it had something to do with orb.reap() blocking forever
# on comms failures; maybe we could create our own orbreapthr
# implementation?
with OrbreapThr(*args, timeout=1, queuesize=orbreapthr_queuesize) as orbreapthr:
log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
self.reject))
threadpool = ThreadPool(maxsize=1)
try:
while True:
try:
success, value = threadpool.spawn(
wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
timestamp = datetime.utcnow()
if not success:
raise value
except (Timeout, NoData), e:
log.debug("orbreapthr.get exception %r" % type(e))
pass
else:
if value is None:
raise Exception('Nothing to publish')
self._publish(value, timestamp)
finally:
# This blocks until all threads in the pool return. That's
# critical; if the orbreapthr dies before the get thread,
# segfaults ensue.
threadpool.kill()
except Exception, e:
log.error("OrbPktSrc terminating due to exception", exc_info=True)
raise
def bench_apply(loops):
pool = ThreadPool(1)
t0 = perf.perf_counter()
for _ in xrange(loops):
for _ in xrange(N):
pool.apply(noop)
pool.join()
pool.kill()
return perf.perf_counter() - t0
def bench_apply(loops):
pool = ThreadPool(1)
t0 = perf.perf_counter()
for _ in xrange(loops):
for _ in xrange(N):
pool.apply(noop)
pool.join()
pool.kill()
return perf.perf_counter() - t0
def bench_spawn_wait(loops):
pool = ThreadPool(1)
t0 = perf.perf_counter()
for _ in xrange(loops):
for _ in xrange(N):
r = pool.spawn(noop)
r.get()
pool.join()
pool.kill()
return perf.perf_counter() - t0
def bench_spawn_wait(loops):
pool = ThreadPool(1)
t0 = perf.perf_counter()
for _ in xrange(loops):
for _ in xrange(N):
r = pool.spawn(noop)
r.get()
pool.join()
pool.kill()
return perf.perf_counter() - t0
def test(self):
pool = ThreadPool(1)
pool.spawn(noop)
gevent.sleep(0)
pool.kill()
def test(self):
pool = ThreadPool(1)
pool.spawn(func)
gevent.sleep(0)
pool.kill()
