def test_exceptionpreservation(self):
# events for controlling execution order
gt1event = eventlet.Event()
gt2event = eventlet.Event()
def test_gt1():
try:
raise KeyError()
except KeyError:
gt1event.send('exception')
gt2event.wait()
assert sys.exc_info()[0] is KeyError
gt1event.send('test passed')
def test_gt2():
gt1event.wait()
gt1event.reset()
assert sys.exc_info()[0] is None
try:
raise ValueError()
except ValueError:
gt2event.send('exception')
gt1event.wait()
assert sys.exc_info()[0] is ValueError
g1 = eventlet.spawn(test_gt1)
g2 = eventlet.spawn(test_gt2)
try:
g1.wait()
g2.wait()
finally:
g1.kill()
g2.kill()
def test_change_subscription(self):
# FIXME: Extensive testing showed this particular test is the root cause
# of sporadic failures on Travis.
pub, sub, port = self.create_bound_pair(zmq.PUB, zmq.SUB)
sub.setsockopt(zmq.SUBSCRIBE, b'test')
eventlet.sleep(0)
sub_ready = eventlet.Event()
sub_last = eventlet.Event()
sub_done = eventlet.Event()
def rx():
while sub.recv() != b'test BEGIN':
eventlet.sleep(0)
sub_ready.send()
count = 0
while True:
msg = sub.recv()
if msg == b'test BEGIN':
# BEGIN may come many times
continue
if msg == b'test LAST':
sub.setsockopt(zmq.SUBSCRIBE, b'done')
sub.setsockopt(zmq.UNSUBSCRIBE, b'test')
eventlet.sleep(0)
# In real application you should either sync
# or tolerate loss of messages.
sub_last.send()
if msg == b'done DONE':
break
count += 1
sub_done.send(count)
def tx():
# Sync receiver ready to avoid loss of first packets
while not sub_ready.ready():
pub.send(b'test BEGIN')
eventlet.sleep(0.005)
for i in range(1, 101):
msg = 'test {0}'.format(i).encode()
if i != 50:
pub.send(msg)
else:
pub.send(b'test LAST')
sub_last.wait()
# XXX: putting a real delay of 1ms here fixes sporadic failures on Travis
# just yield eventlet.sleep(0) doesn't cut it
eventlet.sleep(0.001)
pub.send(b'done DONE')
eventlet.spawn(rx)
eventlet.spawn(tx)
rx_count = sub_done.wait()
self.assertEqual(rx_count, 50)
def test_waiters_get_woken(self):
# verify that when there's someone waiting on an empty pool
# and someone puts an immediately-closed connection back in
# the pool that the waiter gets woken
self.pool.put(self.connection)
self.pool.clear()
self.pool = self.create_pool(max_size=1, max_age=0)
self.connection = self.pool.get()
self.assertEqual(self.pool.free(), 0)
self.assertEqual(self.pool.waiting(), 0)
e = eventlet.Event()
def retrieve(pool, ev):
c = pool.get()
ev.send(c)
eventlet.spawn(retrieve, self.pool, e)
eventlet.sleep(0) # these two sleeps should advance the retrieve
eventlet.sleep(0) # coroutine until it's waiting in get()
self.assertEqual(self.pool.free(), 0)
self.assertEqual(self.pool.waiting(), 1)
self.pool.put(self.connection)
timer = eventlet.Timeout(1)
conn = e.wait()
timer.cancel()
self.assertEqual(self.pool.free(), 0)
self.assertEqual(self.pool.waiting(), 0)
self.pool.put(conn)
def test_waiting(self):
pool = eventlet.GreenPool(1)
done = eventlet.Event()
def consume():
done.wait()
def waiter(pool):
gt = pool.spawn(consume)
gt.wait()
waiters = []
self.assertEqual(pool.running(), 0)
waiters.append(eventlet.spawn(waiter, pool))
eventlet.sleep(0)
self.assertEqual(pool.waiting(), 0)
waiters.append(eventlet.spawn(waiter, pool))
eventlet.sleep(0)
self.assertEqual(pool.waiting(), 1)
waiters.append(eventlet.spawn(waiter, pool))
eventlet.sleep(0)
self.assertEqual(pool.waiting(), 2)
self.assertEqual(pool.running(), 1)
done.send(None)
for w in waiters:
w.wait()
self.assertEqual(pool.waiting(), 0)
self.assertEqual(pool.running(), 0)
def test_send_1k_req_rep(self):
req, rep, port = self.create_bound_pair(zmq.REQ, zmq.REP)
eventlet.sleep()
done = eventlet.Event()
def tx():
tx_i = 0
req.send(str(tx_i).encode())
while req.recv() != b'done':
tx_i += 1
req.send(str(tx_i).encode())
done.send(0)
def rx():
while True:
rx_i = rep.recv()
if rx_i == b"1000":
rep.send(b'done')
break
rep.send(b'i')
eventlet.spawn(tx)
eventlet.spawn(rx)
final_i = done.wait()
self.assertEqual(final_i, 0)
def test_reset(self):
evt = eventlet.Event()
# calling reset before send should throw
self.assertRaises(AssertionError, evt.reset)
value = 'some stuff'
def send_to_event():
evt.send(value)
eventlet.spawn_n(send_to_event)
self.assertEqual(evt.wait(), value)
# now try it again, and we should get the same exact value,
# and we shouldn't be allowed to resend without resetting
value2 = 'second stuff'
self.assertRaises(AssertionError, evt.send, value2)
self.assertEqual(evt.wait(), value)
# reset and everything should be happy
evt.reset()
def send_to_event2():
evt.send(value2)
eventlet.spawn_n(send_to_event2)
self.assertEqual(evt.wait(), value2)
def spawn(self, function, *args, **kwargs):
"""Run the *function* with its arguments in its own green thread.
Returns the :class:`GreenThread <eventlet.GreenThread>`
object that is running the function, which can be used to retrieve the
results.
If the pool is currently at capacity, ``spawn`` will block until one of
the running greenthreads completes its task and frees up a slot.
This function is reentrant; *function* can call ``spawn`` on the same
pool without risk of deadlocking the whole thing.
"""
# if reentering an empty pool, don't try to wait on a coroutine freeing
# itself -- instead, just execute in the current coroutine
current = eventlet.getcurrent()
if self.sem.locked() and current in self.coroutines_running:
# a bit hacky to use the GT without switching to it
gt = eventlet.greenthread.GreenThread(current)
gt.main(function, args, kwargs)
return gt
else:
self.sem.acquire()
gt = eventlet.spawn(function, *args, **kwargs)
if not self.coroutines_running:
self.no_coros_running = eventlet.Event()
self.coroutines_running.add(gt)
gt.link(self._spawn_done)
return gt
def assert_pool_has_free(self, pool, num_free):
self.assertEqual(pool.free(), num_free)
def wait_long_time(e):
e.wait()
timer = eventlet.Timeout(1)
try:
evt = eventlet.Event()
for x in six.moves.range(num_free):
pool.spawn(wait_long_time, evt)
# if the pool has fewer free than we expect,
# then we'll hit the timeout error
finally:
timer.cancel()
# if the runtime error is not raised it means the pool had
# some unexpected free items
timer = eventlet.Timeout(0, RuntimeError)
try:
self.assertRaises(RuntimeError, pool.spawn, wait_long_time, evt)
finally:
timer.cancel()
# clean up by causing all the wait_long_time functions to return
evt.send(None)
eventlet.sleep(0)
eventlet.sleep(0)
def test_send_during_recv(self):
sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ)
eventlet.sleep()
num_recvs = 30
done_evts = [eventlet.Event() for _ in range(num_recvs)]
def slow_rx(done, msg):
self.assertEqual(sender.recv(), msg)
done.send(0)
def tx():
tx_i = 0
while tx_i <= 1000:
sender.send(str(tx_i).encode())
tx_i += 1
def rx():
while True:
rx_i = receiver.recv()
if rx_i == b"1000":
for i in range(num_recvs):
receiver.send(('done%d' % i).encode())
eventlet.sleep()
return
for i in range(num_recvs):
eventlet.spawn(slow_rx, done_evts[i], ("done%d" % i).encode())
eventlet.spawn(tx)
eventlet.spawn(rx)
for evt in done_evts:
self.assertEqual(evt.wait(), 0)
def test_exponential_backoff_subscribe_error(
self,
entrypoint,
config,
mock_pubsub,
mock_sleep,
backoff_factor,
sleep_durations,
):
config['REDIS'] = {'pubsub_backoff_factor': backoff_factor}
entrypoint.setup()
event = eventlet.Event()
mock_pubsub.psubscribe.side_effect = [
ConnectionError('Error1'),
Exception('Error2'),
ConnectionError('Error3'),
TimeoutError('Error4'),
None,
None, # Subscribes successfully
]
mock_pubsub.listen.side_effect = event.wait
with eventlet.Timeout(TIMEOUT):
entrypoint.start()
sleep(TIME_SLEEP)
entrypoint.stop()
assert mock_sleep.call_args_list == [
call(duration) for duration in sleep_durations
]
def test_recv_during_send(self):
sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ)
eventlet.sleep()
done = eventlet.Event()
try:
SNDHWM = zmq.SNDHWM
except AttributeError:
# ZeroMQ <3.0
SNDHWM = zmq.HWM
sender.setsockopt(SNDHWM, 10)
sender.setsockopt(zmq.SNDBUF, 10)
receiver.setsockopt(zmq.RCVBUF, 10)
def tx():
tx_i = 0
while tx_i <= 1000:
sender.send(str(tx_i).encode())
tx_i += 1
done.send(0)
eventlet.spawn(tx)
final_i = done.wait()
self.assertEqual(final_i, 0)
def test_default_backoff_factor_listen_error(self, entrypoint, config,
mock_pubsub, mock_sleep):
config['REDIS'].pop('pubsub_backoff_factor', None)
entrypoint.setup()
event = eventlet.Event()
mock_pubsub.listen.side_effect = [
ConnectionError('Error1'),
Exception('Error2'),
ConnectionError('Error3'),
TimeoutError('Error4'),
redis_listen(
{
'type': 'pmessage',
'pattern': '__keyspace@*__:*',
'channel': '__keyspace@0__:foo',
'data': 'expire',
},
ConnectionError('Error5'),
),
redis_listen(event.wait),
]
with eventlet.Timeout(TIMEOUT):
entrypoint.start()
sleep(TIME_SLEEP)
entrypoint.stop()
assert mock_sleep.call_args_list == [
call(2),
call(4),
call(8),
call(16),
call(2),
]
def test_error_subscribing(self, mock_redis_client, entrypoint):
mock_pubsub_1 = MagicMock()
mock_pubsub_2 = MagicMock()
mock_redis_client.pubsub.side_effect = [mock_pubsub_1, mock_pubsub_2]
mock_pubsub_1.psubscribe.side_effect = [None, ConnectionError('Boom!')]
event = eventlet.Event()
mock_pubsub_2.psubscribe.return_value = None
mock_pubsub_2.listen.side_effect = event.wait
entrypoint.setup()
with eventlet.Timeout(TIMEOUT):
entrypoint.start()
sleep(TIME_SLEEP)
entrypoint.stop()
assert mock_redis_client.pubsub.call_args_list == [call(), call()]
assert mock_pubsub_1.psubscribe.call_args_list == [
call('__keyevent@0__:*'),
call('__keyspace@0__:*'),
]
assert not mock_pubsub_1.listen.called
assert mock_pubsub_2.psubscribe.call_args_list == [
call('__keyevent@0__:*'),
call('__keyspace@0__:*'),
]
assert mock_pubsub_2.listen.call_args_list == [call()]
def test_resize(self):
pool = eventlet.GreenPool(2)
evt = eventlet.Event()
def wait_long_time(e):
e.wait()
pool.spawn(wait_long_time, evt)
pool.spawn(wait_long_time, evt)
self.assertEqual(pool.free(), 0)
self.assertEqual(pool.running(), 2)
self.assert_pool_has_free(pool, 0)
# verify that the pool discards excess items put into it
pool.resize(1)
# cause the wait_long_time functions to return, which will
# trigger puts to the pool
evt.send(None)
eventlet.sleep(0)
eventlet.sleep(0)
self.assertEqual(pool.free(), 1)
self.assertEqual(pool.running(), 0)
self.assert_pool_has_free(pool, 1)
# resize larger and assert that there are more free items
pool.resize(2)
self.assertEqual(pool.free(), 2)
self.assertEqual(pool.running(), 0)
self.assert_pool_has_free(pool, 2)
def test_send_1k_pub_sub(self):
pub, sub_all, port = self.create_bound_pair(zmq.PUB, zmq.SUB)
sub1 = self.context.socket(zmq.SUB)
sub2 = self.context.socket(zmq.SUB)
self.sockets.extend([sub1, sub2])
addr = 'tcp://127.0.0.1:%s' % port
sub1.connect(addr)
sub2.connect(addr)
sub_all.setsockopt(zmq.SUBSCRIBE, b'')
sub1.setsockopt(zmq.SUBSCRIBE, b'sub1')
sub2.setsockopt(zmq.SUBSCRIBE, b'sub2')
sub_all_done = eventlet.Event()
sub1_done = eventlet.Event()
sub2_done = eventlet.Event()
eventlet.sleep(0.2)
def rx(sock, done_evt, msg_count=10000):
count = 0
while count < msg_count:
msg = sock.recv()
eventlet.sleep()
if b'LAST' in msg:
break
count += 1
done_evt.send(count)
def tx(sock):
for i in range(1, 1001):
msg = ("sub%s %s" % ([2, 1][i % 2], i)).encode()
sock.send(msg)
eventlet.sleep()
sock.send(b'sub1 LAST')
sock.send(b'sub2 LAST')
eventlet.spawn(rx, sub_all, sub_all_done)
eventlet.spawn(rx, sub1, sub1_done)
eventlet.spawn(rx, sub2, sub2_done)
eventlet.spawn(tx, pub)
sub1_count = sub1_done.wait()
sub2_count = sub2_done.wait()
sub_all_count = sub_all_done.wait()
self.assertEqual(sub1_count, 500)
self.assertEqual(sub2_count, 500)
self.assertEqual(sub_all_count, 1000)
def test_double_exception(self):
evt = eventlet.Event()
# send an exception through the event
evt.send(exc=RuntimeError('from test_double_exception'))
self.assertRaises(RuntimeError, evt.wait)
evt.reset()
# shouldn't see the RuntimeError again
eventlet.Timeout(0.001)
self.assertRaises(eventlet.Timeout, evt.wait)
def do_flush(self):
if not self.payload_buffer:
return
buffer_switch_event = eventlet.Event()
eventlet.spawn_n(self._gt_flush, buffer_switch_event)
buffer_switch_event.wait()
self.payload_buffer = []
self.total_buffer_size = 0
def test_close_during_recv(self):
sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ)
eventlet.sleep()
done1 = eventlet.Event()
done2 = eventlet.Event()
def rx(e):
self.assertRaisesErrno(RECV_ON_CLOSED_SOCKET_ERRNOS, receiver.recv)
e.send()
eventlet.spawn(rx, done1)
eventlet.spawn(rx, done2)
eventlet.sleep()
receiver.close()
done1.wait()
done2.wait()
def test_waiting_for_event(self):
evt = eventlet.Event()
value = 'some stuff'
def send_to_event():
evt.send(value)
eventlet.spawn_n(send_to_event)
self.assertEqual(evt.wait(), value)
def test_wait_timeout_exceed():
evt = eventlet.Event()
delay = 0.1
eventlet.spawn_after(delay * 2, evt.send, True)
t1 = eventlet.hubs.get_hub().clock()
with eventlet.Timeout(delay, False):
result = evt.wait(timeout=delay)
td = eventlet.hubs.get_hub().clock() - t1
assert not result
assert td >= delay
def test_does_not_use_notification_events_config_if_not_provided(
self, create_service, config, mock_redis_client, mock_pubsub):
event = eventlet.Event()
mock_pubsub.listen.side_effect = event.wait
config['REDIS'].pop('notification_events')
create_service(uri_config_key=URI_CONFIG_KEY,
events='*',
keys='*',
dbs='*')
assert mock_redis_client.config_set.call_args_list == []
def test_stop_entrypoint(self, mock_pubsub, entrypoint, method):
event = eventlet.Event()
mock_pubsub.listen.side_effect = event.wait
stop_method = getattr(entrypoint, method)
entrypoint.setup()
with eventlet.Timeout(TIMEOUT):
entrypoint.start()
sleep(TIME_SLEEP)
stop_method()
assert entrypoint._thread.dead is True
def __init__(self, size=1000):
try:
size = int(size)
except ValueError as e:
msg = 'GreenPool() expect size :: int, actual: {0} {1}'.format(type(size), str(e))
raise TypeError(msg)
if size < 0:
msg = 'GreenPool() expect size >= 0, actual: {0}'.format(repr(size))
raise ValueError(msg)
self.size = size
self.coroutines_running = set()
self.sem = eventlet.Semaphore(size)
self.no_coros_running = eventlet.Event()
def test_uses_notification_events_config_if_provided(
self, create_service, config, mock_redis_client, mock_pubsub):
event = eventlet.Event()
mock_pubsub.listen.side_effect = event.wait
config['REDIS']['notification_events'] = 'test_value'
create_service(uri_config_key=URI_CONFIG_KEY,
events='*',
keys='*',
dbs='*')
assert mock_redis_client.config_set.call_args_list == [
call('notify-keyspace-events', 'test_value')
]
def test_killall_remaining_results(self):
semaphore = eventlet.Event()
def native_fun():
time.sleep(.5)
def gt_fun():
semaphore.send(None)
tpool.execute(native_fun)
gt = eventlet.spawn(gt_fun)
semaphore.wait()
tpool.killall()
gt.wait()
def _test_multiple_waiters(self, exception):
evt = eventlet.Event()
results = []
def wait_on_event(i_am_done):
evt.wait()
results.append(True)
i_am_done.send()
if exception:
raise Exception()
waiters = []
count = 5
for i in range(count):
waiters.append(eventlet.Event())
eventlet.spawn_n(wait_on_event, waiters[-1])
eventlet.sleep() # allow spawns to start executing
evt.send()
for w in waiters:
w.wait()
self.assertEqual(len(results), count)
def test_block(self):
e = zmq._BlockedThread()
done = eventlet.Event()
self.assertFalse(e)
def block():
e.block()
done.send(1)
eventlet.spawn(block)
eventlet.sleep()
self.assertFalse(done.has_result())
e.wake()
done.wait()
def test_recv_spawned_before_send_is_non_blocking(self):
req, rep, port = self.create_bound_pair(zmq.PAIR, zmq.PAIR)
# req.connect(ipc)
# rep.bind(ipc)
eventlet.sleep()
msg = dict(res=None)
done = eventlet.Event()
def rx():
msg['res'] = rep.recv()
done.send('done')
eventlet.spawn(rx)
req.send(b'test')
done.wait()
self.assertEqual(msg['res'], b'test')
def spawn_n(self, function, *args, **kwargs):
"""Create a greenthread to run the *function*, the same as
:meth:`spawn`. The difference is that :meth:`spawn_n` returns
None; the results of *function* are not retrievable.
"""
# if reentering an empty pool, don't try to wait on a coroutine freeing
# itself -- instead, just execute in the current coroutine
current = eventlet.getcurrent()
if self.sem.locked() and current in self.coroutines_running:
self._spawn_n_impl(function, args, kwargs, None)
else:
self.sem.acquire()
g = eventlet.spawn_n(self._spawn_n_impl, function, args, kwargs,
True)
if not self.coroutines_running:
self.no_coros_running = eventlet.Event()
self.coroutines_running.add(g)
def test_multiple_coros(self):
evt = eventlet.Event()
results = []
def producer():
results.append('prod')
evt.send()
def consumer():
results.append('cons1')
evt.wait()
results.append('cons2')
pool = eventlet.GreenPool(2)
done = pool.spawn(consumer)
pool.spawn_n(producer)
done.wait()
self.assertEqual(['cons1', 'prod', 'cons2'], results)