class ThreadPool(object):
def __init__(self, concurrency, handler, generator, finalizer=None):
self.concurrency = concurrency
self.handler = handler
self.generator = generator
self.finalizer = finalizer
def handler_wrap(self):
try:
label("thread-pool-%s" % self.handler)
while True:
self.waiting += 1
if self.waiting == 1:
self.trigger.set()
i = self.q.get()
self.waiting -= 1
if i == ThreadPoolDie:
return
self.handler(i)
finally:
self.running -=1
if self.waiting == 0:
self.trigger.set()
if self.running == 0:
self.finished.set()
def __call__(self):
self.q = Queue()
self.trigger = Event()
self.finished = Event()
self.waiting = 0
self.running = 0
try:
while True:
for x in xrange(self.concurrency - self.running):
self.running += 1
fork(self.handler_wrap)
if self.waiting == 0:
self.trigger.wait()
self.trigger.clear()
try:
n = self.generator()
except StopIteration:
break
self.q.put(n)
sleep()
finally:
for x in xrange(self.concurrency):
self.q.put(ThreadPoolDie)
if self.finalizer:
self.finished.wait()
fork(self.finalizer)
class ThreadPool(object):
def __init__(self, concurrency, handler, generator, finalizer=None):
self.concurrency = concurrency
self.handler = handler
self.generator = generator
self.finalizer = finalizer
def handler_wrap(self):
try:
label("thread-pool-%s" % self.handler)
while True:
self.waiting += 1
if self.waiting == 1:
self.trigger.set()
i = self.q.get()
self.waiting -= 1
if i == ThreadPoolDie:
return
self.handler(i)
finally:
self.running -= 1
if self.waiting == 0:
self.trigger.set()
if self.running == 0:
self.finished.set()
def __call__(self):
self.q = Queue()
self.trigger = Event()
self.finished = Event()
self.waiting = 0
self.running = 0
try:
while True:
for x in xrange(self.concurrency - self.running):
self.running += 1
fork(self.handler_wrap)
if self.waiting == 0:
self.trigger.wait()
self.trigger.clear()
try:
n = self.generator()
except StopIteration:
break
self.q.put(n)
sleep()
finally:
for x in xrange(self.concurrency):
self.q.put(ThreadPoolDie)
if self.finalizer:
self.finished.wait()
fork(self.finalizer)
class DieselNitroSocket(Waiter):
def __init__(self, bind=None, connect=None, **kwargs):
Waiter.__init__(self)
self.destroyed = False
kwargs['want_eventfd'] = 1
self.socket = pynitro.NitroSocket(**kwargs)
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.fileno())
self.read_gate = Event()
self.hub.register(self.fd, self.messages_exist, self.error, self.error)
self.sent = 0
self.received = 0
def _send_op(self, op):
while True:
try:
op()
except pynitro.NitroFull:
diesel.sleep(0.2)
else:
self.sent += 1
return
def recv(self):
while True:
try:
m = self.socket.recv(self.socket.NOWAIT)
except pynitro.NitroEmpty:
self.read_gate.clear()
self.read_gate.wait()
else:
self.received += 1
return m
def send(self, frame, flags=0):
return self._send_op(
partial(self.socket.send, frame, self.socket.NOWAIT | flags))
def reply(self, orig, frame, flags=0):
return self._send_op(
partial(self.socket.reply, orig, frame, self.socket.NOWAIT | flags))
def process_fire(self, dc):
if not self._early_value:
got = self.ready_early()
if not got:
raise StopWaitDispatch()
assert self._early_value
v = self._early_value
self._early_value = None
return v
def ready_early(self):
if self._early_value:
return True
try:
m = self.socket.recv(self.socket.NOWAIT)
except pynitro.NitroEmpty:
self.read_gate.clear()
return False
else:
self.received += 1
self._early_value = m
return True
def messages_exist(self):
'''Handle state change.
'''
self.read_gate.set()
diesel.fire(self)
def error(self):
raise RuntimeError("OH NOES, some weird nitro FD callback")
def destroy(self):
if not self.destroyed:
self.hub.unregister(self.fd)
del self.socket
self.destroyed = True
def __enter__(self):
return self
def __del__(self):
self.destroy()
def __exit__(self, *args):
self.destroy()
class DieselNitroSocket(Waiter):
def __init__(self, bind=None, connect=None, **kwargs):
Waiter.__init__(self)
self.destroyed = False
kwargs['want_eventfd'] = 1
self.socket = pynitro.NitroSocket(**kwargs)
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.fileno())
self.read_gate = Event()
self.hub.register(self.fd, self.messages_exist, self.error, self.error)
self.sent = 0
self.received = 0
def _send_op(self, op):
while True:
try:
op()
except pynitro.NitroFull:
diesel.sleep(0.2)
else:
self.sent += 1
return
def recv(self):
while True:
try:
m = self.socket.recv(self.socket.NOWAIT)
except pynitro.NitroEmpty:
self.read_gate.clear()
self.read_gate.wait()
else:
self.received += 1
return m
def send(self, frame, flags=0):
return self._send_op(
partial(self.socket.send, frame, self.socket.NOWAIT | flags))
def reply(self, orig, frame, flags=0):
return self._send_op(
partial(self.socket.reply, orig, frame,
self.socket.NOWAIT | flags))
def process_fire(self, dc):
if not self._early_value:
got = self.ready_early()
if not got:
raise StopWaitDispatch()
assert self._early_value
v = self._early_value
self._early_value = None
return v
def ready_early(self):
if self._early_value:
return True
try:
m = self.socket.recv(self.socket.NOWAIT)
except pynitro.NitroEmpty:
self.read_gate.clear()
return False
else:
self.received += 1
self._early_value = m
return True
def messages_exist(self):
'''Handle state change.
'''
self.read_gate.set()
diesel.fire(self)
def error(self):
raise RuntimeError("OH NOES, some weird nitro FD callback")
def destroy(self):
if not self.destroyed:
self.hub.unregister(self.fd)
del self.socket
self.destroyed = True
def __enter__(self):
return self
def __del__(self):
self.destroy()
def __exit__(self, *args):
self.destroy()
