def __call__(self):
self.q = Queue()
self.trigger = Event()
self.finished = Event()
self.waiting = 0
self.running = 0
try:
while True:
for x in xrange(self.concurrency - self.running):
self.running += 1
fork(self.handler_wrap)
if self.waiting == 0:
self.trigger.wait()
self.trigger.clear()
try:
n = self.generator()
except StopIteration:
break
self.q.put(n)
sleep()
finally:
for x in xrange(self.concurrency):
self.q.put(ThreadPoolDie)
if self.finalizer:
self.finished.wait()
fork(self.finalizer)
class TestQueueTimeouts(object):
def setup(self):
self.result = Event()
self.queue = Queue()
self.timeouts = 0
diesel.fork(self.consumer, 0.01)
diesel.fork(self.producer, 0.05)
diesel.fork(self.consumer, 0.10)
ev, val = diesel.first(sleep=TIMEOUT, waits=[self.result])
if ev == 'sleep':
assert 0, 'timed out'
def consumer(self, timeout):
try:
self.queue.get(timeout=timeout)
self.result.set()
except QueueTimeout:
self.timeouts += 1
def producer(self, delay):
diesel.sleep(delay)
self.queue.put('test')
def test_a_consumer_timed_out(self):
assert self.timeouts == 1
def test_a_consumer_got_a_value(self):
assert self.result.is_set
def __init__(self,
socket,
bind=None,
connect=None,
context=None,
linger_time=1000):
self.zctx = context or zctx
self.socket = socket
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.getsockopt(zmq.FD))
self.write_gate = Event()
self.read_gate = Event()
self.linger_time = linger_time
self.hub.register(self.fd, self.handle_transition, self.error,
self.error)
self.handle_transition()
self.destroyed = False
self.sent = 0
self.received = 0
class TestQueueTimeouts(object):
def setup(self):
self.result = Event()
self.queue = Queue()
self.timeouts = 0
diesel.fork(self.consumer, 0.01)
diesel.fork(self.producer, 0.05)
diesel.fork(self.consumer, 0.10)
ev, val = diesel.first(sleep=TIMEOUT, waits=[self.result])
if ev == 'sleep':
assert 0, 'timed out'
def consumer(self, timeout):
try:
self.queue.get(timeout=timeout)
self.result.set()
except QueueTimeout:
self.timeouts += 1
def producer(self, delay):
diesel.sleep(delay)
self.queue.put('test')
def test_a_consumer_timed_out(self):
assert self.timeouts == 1
def test_a_consumer_got_a_value(self):
assert self.result.is_set
def __call__(self):
self.q = Queue()
self.trigger = Event()
self.finished = Event()
self.waiting = 0
self.running = 0
try:
while True:
for x in xrange(self.concurrency - self.running):
self.running += 1
fork(self.handler_wrap)
if self.waiting == 0:
self.trigger.wait()
self.trigger.clear()
try:
n = self.generator()
except StopIteration:
break
self.q.put(n)
sleep()
finally:
for x in xrange(self.concurrency):
self.q.put(ThreadPoolDie)
if self.finalizer:
self.finished.wait()
fork(self.finalizer)
def test_overwriting_custom_signal_handler_fails():
readings = []
success = Event()
failure = Event()
def append_reading(sig, frame):
readings.append(sig)
signal.signal(signal.SIGUSR1, signal.SIG_DFL)
signal.signal(signal.SIGUSR1, append_reading)
def overwriter():
try:
diesel.signal(signal.SIGUSR1)
except diesel.ExistingSignalHandler:
success.set()
else:
failure.set()
diesel.fork(overwriter)
diesel.sleep()
os.kill(os.getpid(), signal.SIGUSR1)
evt, _ = diesel.first(waits=[success, failure])
assert evt is success
assert readings
def setup(self):
self.result = Event()
self.queue = Queue()
self.timeouts = 0
diesel.fork(self.consumer, 0.01)
diesel.fork(self.producer, 0.05)
diesel.fork(self.consumer, 0.10)
ev, val = diesel.first(sleep=TIMEOUT, waits=[self.result])
if ev == 'sleep':
assert 0, 'timed out'
class DieselZMQSocket(Waiter):
'''Integrate zmq's super fast event loop with ours.
'''
def __init__(self,
socket,
bind=None,
connect=None,
context=None,
linger_time=1000):
self.zctx = context or zctx
self.socket = socket
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.getsockopt(zmq.FD))
self.write_gate = Event()
self.read_gate = Event()
self.linger_time = linger_time
self.hub.register(self.fd, self.handle_transition, self.error,
self.error)
self.handle_transition()
self.destroyed = False
self.sent = 0
self.received = 0
def send(self, message, flags=0):
while True:
try:
self.socket.send(message, zmq.NOBLOCK | flags)
except zmq.ZMQError, e:
if e.errno == EAGAIN:
self.handle_transition(
True) # force re-evaluation of EVENTS
self.write_gate.wait()
else:
raise
else:
self.handle_transition(True)
self.sent += 1
break
class ThreadPool(object):
def __init__(self, concurrency, handler, generator):
self.concurrency = concurrency
self.handler = handler
self.generator = generator
def handler_wrap(self):
try:
label("thread-pool-%s" % self.handler)
while True:
self.waiting += 1
if self.waiting == 1:
self.trigger.set()
i = self.q.get()
self.waiting -= 1
if i == ThreadPoolDie:
return
self.handler(i)
finally:
self.running -=1
if self.waiting == 0:
self.trigger.set()
def __call__(self):
self.q = Queue()
self.trigger = Event()
self.waiting = 0
self.running = 0
try:
while True:
for x in xrange(self.concurrency - self.running):
self.running += 1
fork(self.handler_wrap)
if self.waiting == 0:
self.trigger.wait()
self.trigger.clear()
try:
n = self.generator()
except StopIteration:
break
self.q.put(n)
sleep()
finally:
for x in xrange(self.concurrency):
self.q.put(ThreadPoolDie)
class ThreadPool(object):
def __init__(self, concurrency, handler, generator):
self.concurrency = concurrency
self.handler = handler
self.generator = generator
def handler_wrap(self):
try:
label("thread-pool-%s" % self.handler)
while True:
self.waiting += 1
if self.waiting == 1:
self.trigger.set()
i = self.q.get()
self.waiting -= 1
if i == ThreadPoolDie:
return
self.handler(i)
finally:
self.running -= 1
if self.waiting == 0:
self.trigger.set()
def __call__(self):
self.q = Queue()
self.trigger = Event()
self.waiting = 0
self.running = 0
try:
while True:
for x in xrange(self.concurrency - self.running):
self.running += 1
fork(self.handler_wrap)
if self.waiting == 0:
self.trigger.wait()
self.trigger.clear()
try:
n = self.generator()
except StopIteration:
break
self.q.put(n)
sleep()
finally:
for x in xrange(self.concurrency):
self.q.put(ThreadPoolDie)
def __init__(self, socket, bind=None, connect=None, context=None, linger_time=1000):
self.zctx = context or zctx
self.socket = socket
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.getsockopt(zmq.FD))
self.write_gate = Event()
self.read_gate = Event()
self.linger_time = linger_time
self.hub.register(self.fd, self.handle_transition, self.error, self.error)
self.handle_transition()
self.destroyed = False
self.sent = 0
self.received = 0
def setup(self):
self.result = Event()
self.queue = Queue()
self.timeouts = 0
diesel.fork(self.consumer, 0.01)
diesel.fork(self.producer, 0.05)
diesel.fork(self.consumer, 0.10)
ev, val = diesel.first(sleep=TIMEOUT, waits=[self.result])
if ev == 'sleep':
assert 0, 'timed out'
class DieselZMQSocket(Waiter):
'''Integrate zmq's super fast event loop with ours.
'''
def __init__(self, socket, bind=None, connect=None, context=None, linger_time=1000):
self.zctx = context or zctx
self.socket = socket
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.getsockopt(zmq.FD))
self.write_gate = Event()
self.read_gate = Event()
self.linger_time = linger_time
self.hub.register(self.fd, self.handle_transition, self.error, self.error)
self.handle_transition()
self.destroyed = False
self.sent = 0
self.received = 0
def send(self, message, flags=0):
while True:
try:
self.socket.send(message, zmq.NOBLOCK | flags)
except zmq.ZMQError, e:
if e.errno == EAGAIN:
self.handle_transition(True) # force re-evaluation of EVENTS
self.write_gate.wait()
else:
raise
else:
self.handle_transition(True)
self.sent += 1
break
def __init__(self, bind=None, connect=None, **kwargs):
Waiter.__init__(self)
self.destroyed = False
kwargs['want_eventfd'] = 1
self.socket = pynitro.NitroSocket(**kwargs)
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.fileno())
self.read_gate = Event()
self.hub.register(self.fd, self.messages_exist, self.error, self.error)
self.sent = 0
self.received = 0
def test_basic_fork():
v = [0]
done = Event()
def parent():
fork(tottering_child, v)
sleep(0.1)
done.set()
runtime.current_app.add_loop(Loop(parent))
ev, _ = first(sleep=1, waits=[done])
if ev == 'sleep':
assert 0, "timed out"
assert (v[0] == 1)
def test_signals_are_handled_while_event_loop_is_blocked():
done = Event()
def handler():
diesel.signal(signal.SIGUSR1)
done.set()
def killer():
time.sleep(0.1)
os.kill(os.getpid(), signal.SIGUSR1)
diesel.fork(handler)
diesel.thread(killer)
diesel.sleep()
evt, _ = diesel.first(sleep=1, waits=[done])
assert evt is done
def handle_blockon(self, timeout, *args):
valmap = dict(izip(args[0::2], args[1::2]))
timeout = float(timeout)
blocktimer = Timer(timeout, start=False)
blockevent = Event()
while True:
for v, pid in valmap.iteritems():
if v in store and store[v].proposal_id != pid:
return self.send_header("BLOCKON-DONE", v)
if blocktimer.is_due:
self.send_header("BLOCKON-TIMEOUT")
return
for v in valmap:
store.watch(v, blockevent)
ev, _ = first(sleep=blocktimer.till_due, waits=[blockevent])
def test_fire_miss():
done = Event()
v = [0]
def w():
while True:
wait("boom!")
v[0] += 1
def f():
sleep(0.05)
fire("fizz!")
done.set()
fork(f)
fork(w)
fork(w)
ev, _ = first(sleep=1, waits=[done])
assert v[0] == 0 # should not have woken up!
def test_basic_fire():
done = Event()
v = [0]
def w():
while True:
wait("boom!")
v[0] += 1
def f():
sleep(0.05)
fire("boom!")
sleep(0.05)
fire("boom!")
done.set()
fork(f)
fork(w)
ev, _ = first(sleep=1, waits=[done])
assert v[0] == 2
def handle_set(self, key, addition, members, timeout, lock):
members = int(members)
lock = int(lock)
timeout = float(timeout)
start = time.time()
if addition is None:
assert members == 0
assert lock == 0
else:
assert '|' not in addition
assert ':' not in addition
while True:
existing_value = group.network_get(key)
if members and existing_value and existing_value.v is not None:
pfx = existing_value.v
else:
pfx = ''
blocked = False
if members == 0 or pfx.count('|') < members:
blocked = True
pid = group.network_set(self.client_id if lock else None, key,
pfx if addition else None, addition)
if pid:
return self.send_set_response("SET-OKAY", key)
left = (start + timeout) - time.time()
if left <= 0:
return self.send_set_response("SET-TIMEOUT", key)
if blocked and key in store and store[
key].proposal_id != existing_value.proposal_id:
continue # value has changed already, let's try again
trigger = Event()
store.watch(key, trigger)
ev, val = first(sleep=left, waits=[trigger])
if ev != trigger:
return self.send_set_response("SET-TIMEOUT", key)
def __init__(self, bind=None, connect=None, **kwargs):
Waiter.__init__(self)
self.destroyed = False
kwargs['want_eventfd'] = 1
self.socket = pynitro.NitroSocket(**kwargs)
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.fileno())
self.read_gate = Event()
self.hub.register(self.fd, self.messages_exist, self.error, self.error)
self.sent = 0
self.received = 0
def __init__(self, resolver):
Event.__init__(self)
self.error = None
self.value = None
self.resolver = resolver
def __init__(self, resolver):
Event.__init__(self)
self.error = None
self.value = None
self.resolver = resolver
class DieselNitroSocket(Waiter):
def __init__(self, bind=None, connect=None, **kwargs):
Waiter.__init__(self)
self.destroyed = False
kwargs['want_eventfd'] = 1
self.socket = pynitro.NitroSocket(**kwargs)
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.fileno())
self.read_gate = Event()
self.hub.register(self.fd, self.messages_exist, self.error, self.error)
self.sent = 0
self.received = 0
def _send_op(self, op):
while True:
try:
op()
except pynitro.NitroFull:
diesel.sleep(0.2)
else:
self.sent += 1
return
def recv(self):
while True:
try:
m = self.socket.recv(self.socket.NOWAIT)
except pynitro.NitroEmpty:
self.read_gate.clear()
self.read_gate.wait()
else:
self.received += 1
return m
def send(self, frame, flags=0):
return self._send_op(
partial(self.socket.send, frame, self.socket.NOWAIT | flags))
def reply(self, orig, frame, flags=0):
return self._send_op(
partial(self.socket.reply, orig, frame, self.socket.NOWAIT | flags))
def process_fire(self, dc):
if not self._early_value:
got = self.ready_early()
if not got:
raise StopWaitDispatch()
assert self._early_value
v = self._early_value
self._early_value = None
return v
def ready_early(self):
if self._early_value:
return True
try:
m = self.socket.recv(self.socket.NOWAIT)
except pynitro.NitroEmpty:
self.read_gate.clear()
return False
else:
self.received += 1
self._early_value = m
return True
def messages_exist(self):
'''Handle state change.
'''
self.read_gate.set()
diesel.fire(self)
def error(self):
raise RuntimeError("OH NOES, some weird nitro FD callback")
def destroy(self):
if not self.destroyed:
self.hub.unregister(self.fd)
del self.socket
self.destroyed = True
def __enter__(self):
return self
def __del__(self):
self.destroy()
def __exit__(self, *args):
self.destroy()
from diesel.util.event import Event
from diesel import quickstart, quickstop, sleep
pistol = Event()
def racer():
pistol.wait()
print "CHAAARGE!"
def starter():
print "Ready..."
sleep(1)
print "Set..."
sleep(1)
print " ~~~~~~~~~ BANG ~~~~~~~~~ "
pistol.set()
sleep(1)
print " ~~~~~~~~~ RACE OVER ~~~~~~~~~ "
quickstop()
quickstart(starter, [racer for x in xrange(8)])
class DieselNitroSocket(Waiter):
def __init__(self, bind=None, connect=None, **kwargs):
Waiter.__init__(self)
self.destroyed = False
kwargs['want_eventfd'] = 1
self.socket = pynitro.NitroSocket(**kwargs)
self._early_value = None
from diesel.runtime import current_app
from diesel.hub import IntWrap
if bind:
assert not connect
self.socket.bind(bind)
elif connect:
assert not bind
self.socket.connect(connect)
self.hub = current_app.hub
self.fd = IntWrap(self.socket.fileno())
self.read_gate = Event()
self.hub.register(self.fd, self.messages_exist, self.error, self.error)
self.sent = 0
self.received = 0
def _send_op(self, op):
while True:
try:
op()
except pynitro.NitroFull:
diesel.sleep(0.2)
else:
self.sent += 1
return
def recv(self):
while True:
try:
m = self.socket.recv(self.socket.NOWAIT)
except pynitro.NitroEmpty:
self.read_gate.clear()
self.read_gate.wait()
else:
self.received += 1
return m
def send(self, frame, flags=0):
return self._send_op(
partial(self.socket.send, frame, self.socket.NOWAIT | flags))
def reply(self, orig, frame, flags=0):
return self._send_op(
partial(self.socket.reply, orig, frame,
self.socket.NOWAIT | flags))
def process_fire(self, dc):
if not self._early_value:
got = self.ready_early()
if not got:
raise StopWaitDispatch()
assert self._early_value
v = self._early_value
self._early_value = None
return v
def ready_early(self):
if self._early_value:
return True
try:
m = self.socket.recv(self.socket.NOWAIT)
except pynitro.NitroEmpty:
self.read_gate.clear()
return False
else:
self.received += 1
self._early_value = m
return True
def messages_exist(self):
'''Handle state change.
'''
self.read_gate.set()
diesel.fire(self)
def error(self):
raise RuntimeError("OH NOES, some weird nitro FD callback")
def destroy(self):
if not self.destroyed:
self.hub.unregister(self.fd)
del self.socket
self.destroyed = True
def __enter__(self):
return self
def __del__(self):
self.destroy()
def __exit__(self, *args):
self.destroy()
