def test_recv_into_timeout(self):
buf = buffer(array.array('B'))
listener = sockets.GreenSocket()
listener.bind(('', 0))
listener.listen(50)
address, port = listener.getsockname()
self.assertNotEquals(address, 0)
accepting = event.Event()
accepted = event.Event()
def server():
# accept the connection in another greenlet
accepting.send()
sock, addr = listener.accept()
accepted.wait()
gt = spawn(server)
client = sockets.GreenSocket()
client.settimeout(0.1)
accepting.wait()
client.connect(('127.0.0.1', port))
try:
client.recv_into(buf, 100)
self.fail("socket.timeout not raised")
except socket.timeout, e:
self.assert_(hasattr(e, 'args'))
self.assertEqual(e.args[0], 'timed out')
def test_recv_into(self):
self.reset_timeout(100000)
listener = sockets.GreenSocket()
listener.bind(('', 0))
listener.listen(50)
address, port = listener.getsockname()
self.assertNotEquals(address, 0)
accepting = event.Event()
received = event.Event()
sent_data = '1234567890'
def server():
# accept the connection in another greenlet
accepting.send()
sock, addr = listener.accept()
sock.send(sent_data)
def client():
buf = buffer(array.array('B'))
client = sockets.GreenSocket()
accepting.wait()
sleep(0.5)
client.connect(('127.0.0.1', port))
client.recv_into(buf, 5000)
received.send(buf)
waitall(spawn(client), spawn(server))
received_data = received.wait()
self.assertEquals(sent_data, received_data)
def test_evy():
event1 = event.Event()
event2 = event.Event()
event1.send()
thread1 = threads.spawn(run, event1, event2)
thread2 = threads.spawn(run, event2, event1)
thread1.wait()
thread2.wait()
def test_send_1k_req_rep(self):
req, rep, port = self.create_bound_pair(zmq.REQ, zmq.REP)
sleep()
done = event.Event()
def tx():
tx_i = 0
req.send(str(tx_i))
while req.recv() != 'done':
tx_i += 1
req.send(str(tx_i))
done.send(0)
def rx():
while True:
rx_i = rep.recv()
if rx_i == "1000":
rep.send('done')
break
rep.send('i')
spawn(tx)
spawn(rx)
final_i = done.wait()
self.assertEqual(final_i, 0)
def test_waiters_get_woken(self):
# verify that when there's someone waiting on an empty pool
# and someone puts an immediately-closed connection back in
# the pool that the waiter gets woken
self.pool.put(self.connection)
self.pool.clear()
self.pool = self.create_pool(max_size=1, max_age=0)
self.connection = self.pool.get()
self.assertEquals(self.pool.free(), 0)
self.assertEquals(self.pool.waiting(), 0)
e = event.Event()
def retrieve(pool, ev):
c = pool.get()
ev.send(c)
evy.spawn(retrieve, self.pool, e)
sleep(0) # these two sleeps should advance the retrieve
sleep(0) # coroutine until it's waiting in get()
self.assertEquals(self.pool.free(), 0)
self.assertEquals(self.pool.waiting(), 1)
self.pool.put(self.connection)
timer = Timeout(1)
conn = e.wait()
timer.cancel()
self.assertEquals(self.pool.free(), 0)
self.assertEquals(self.pool.waiting(), 0)
self.pool.put(conn)
def test_send_during_recv(self):
sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ)
sleep()
num_recvs = 30
done_evts = [event.Event() for _ in range(num_recvs)]
def slow_rx(done, msg):
self.assertEqual(sender.recv(), msg)
done.send(0)
def tx():
tx_i = 0
while tx_i <= 1000:
sender.send(str(tx_i))
tx_i += 1
def rx():
while True:
rx_i = receiver.recv()
if rx_i == "1000":
for i in range(num_recvs):
receiver.send('done%d' % i)
sleep()
return
for i in range(num_recvs):
spawn(slow_rx, done_evts[i], "done%d" % i)
spawn(tx)
spawn(rx)
for evt in done_evts:
self.assertEqual(evt.wait(), 0)
def test_send_during_recv_multipart(self):
sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ)
sleep()
num_recvs = 30
done_evts = [event.Event() for _ in range(num_recvs)]
def slow_rx(done, msg):
self.assertEqual(sender.recv_multipart(), msg)
done.send(0)
def tx():
tx_i = 0
while tx_i <= 1000:
sender.send_multipart([str(tx_i), '1', '2', '3'])
tx_i += 1
def rx():
while True:
rx_i = receiver.recv_multipart()
if rx_i == ["1000", '1', '2', '3']:
for i in range(num_recvs):
receiver.send_multipart(['done%d' % i, 'a', 'b', 'c'])
sleep()
return
for i in range(num_recvs):
spawn(slow_rx, done_evts[i], ["done%d" % i, 'a', 'b', 'c'])
spawn(tx)
spawn(rx)
for i in range(num_recvs):
final_i = done_evts[i].wait()
self.assertEqual(final_i, 0)
def test_sendall_timeout(self):
listener = sockets.GreenSocket()
listener.bind(('', 0))
listener.listen(50)
evt = event.Event()
def server():
# accept the connection in another greenlet
sock, addr = listener.accept()
evt.wait()
gt = spawn(server)
addr = listener.getsockname()
self.assertNotEqual(addr[1], 0)
client = sockets.GreenSocket()
client.settimeout(0.1)
client.connect(addr)
try:
msg = s2b("A") * (8 * 1024 * 1024)
# want to exceed the size of the OS buffer so it'll block
client.sendall(msg)
self.fail("socket.timeout not raised")
except socket.timeout, e:
self.assert_(hasattr(e, 'args'))
self.assertEqual(e.args[1], 'timed out')
def test_send_timeout(self):
self.reset_timeout(1000000)
listener = convenience.listen(('', 0))
_, port = listener.getsockname()
evt = event.Event()
def server():
# accept the connection in another greenlet
sock, addr = listener.accept()
evt.wait()
def client():
client = sockets.GreenSocket()
client.connect(('127.0.0.1', port))
try:
client.settimeout(0.00001)
msg = s2b("A") * (
100000) # large enough number to overwhelm most buffers
total_sent = 0
# want to exceed the size of the OS buffer so it'll block in a
# single send
for x in range(10):
total_sent += client.send(msg)
self.fail("socket.timeout not raised")
except socket.timeout, e:
self.assert_(hasattr(e, 'args'))
self.assertEqual(e.args[1], 'timed out')
finally:
def test_app_socket_errors_76 (self):
error_detected = [False]
done_with_request = event.Event()
site = self.site
def error_detector (environ, start_response):
try:
try:
return site(environ, start_response)
except:
error_detected[0] = True
raise
finally:
done_with_request.send(True)
self.site = error_detector
self.spawn_server()
connect_data = [
"GET /error HTTP/1.1",
"Upgrade: WebSocket",
"Connection: Upgrade",
"Host: localhost:%s" % self.port,
"Origin: http://localhost:%s" % self.port,
"Sec-WebSocket-Protocol: ws",
"Sec-WebSocket-Key1: 4 @1 46546xW%0l 1 5",
"Sec-WebSocket-Key2: 12998 5 Y3 1 .P00",
]
sock = connect(
('localhost', self.port))
sock.sendall('\r\n'.join(connect_data) + '\r\n\r\n^n:ds[4U')
resp = sock.recv(1024)
done_with_request.wait()
self.assert_(error_detected[0])
def spawn (self, function, *args, **kwargs):
"""
Run the *function* with its arguments in its own green thread.
Returns the :class:`GreenThread <evy.greenthread.GreenThread>`
object that is running the function, which can be used to retrieve the
results.
If the pool is currently at capacity, ``spawn`` will block until one of
the running greenthreads completes its task and frees up a slot.
This function is reentrant; *function* can call ``spawn`` on the same
pool without risk of deadlocking the whole thing.
"""
# if reentering an empty pool, don't try to wait on a coroutine freeing
# itself -- instead, just execute in the current coroutine
current = greenthread.getcurrent()
if self.sem.locked() and current in self.coroutines_running:
# a bit hacky to use the GT without switching to it
gt = greenthread.GreenThread(current)
gt.main(function, args, kwargs)
return gt
else:
self.sem.acquire()
gt = greenthread.spawn(function, *args, **kwargs)
if not self.coroutines_running:
self.no_coros_running = event.Event()
self.coroutines_running.add(gt)
gt.link(self._spawn_done)
return gt
def test_breaking_the_connection_75 (self):
error_detected = [False]
done_with_request = event.Event()
site = self.site
def error_detector (environ, start_response):
try:
try:
return site(environ, start_response)
except:
error_detected[0] = True
raise
finally:
done_with_request.send(True)
self.site = error_detector
self.spawn_server()
connect_data = [
"GET /range HTTP/1.1",
"Upgrade: WebSocket",
"Connection: Upgrade",
"Host: localhost:%s" % self.port,
"Origin: http://localhost:%s" % self.port,
"WebSocket-Protocol: ws",
]
sock = connect(
('localhost', self.port))
sock.sendall('\r\n'.join(connect_data) + '\r\n\r\n')
resp = sock.recv(1024) # get the headers
sock.close() # close while the app is running
done_with_request.wait()
self.assert_(not error_detected[0])
def test_send_1k_pub_sub(self):
pub, sub_all, port = self.create_bound_pair(zmq.PUB, zmq.SUB)
sub1 = self.context.socket(zmq.SUB)
sub2 = self.context.socket(zmq.SUB)
self.sockets.extend([sub1, sub2])
addr = 'tcp://127.0.0.1:%s' % port
sub1.connect(addr)
sub2.connect(addr)
sub_all.setsockopt(zmq.SUBSCRIBE, '')
sub1.setsockopt(zmq.SUBSCRIBE, 'sub1')
sub2.setsockopt(zmq.SUBSCRIBE, 'sub2')
sub_all_done = event.Event()
sub1_done = event.Event()
sub2_done = event.Event()
sleep(0.2)
def rx(sock, done_evt, msg_count=10000):
count = 0
while count < msg_count:
msg = sock.recv()
sleep()
if 'LAST' in msg:
break
count += 1
done_evt.send(count)
def tx(sock):
for i in range(1, 1001):
msg = "sub%s %s" % ([2, 1][i % 2], i)
sock.send(msg)
sleep()
sock.send('sub1 LAST')
sock.send('sub2 LAST')
spawn(rx, sub_all, sub_all_done)
spawn(rx, sub1, sub1_done)
spawn(rx, sub2, sub2_done)
spawn(tx, pub)
sub1_count = sub1_done.wait()
sub2_count = sub2_done.wait()
sub_all_count = sub_all_done.wait()
self.assertEqual(sub1_count, 500)
self.assertEqual(sub2_count, 500)
self.assertEqual(sub_all_count, 1000)
def test_close_during_recv(self):
sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ)
sleep()
done1 = event.Event()
done2 = event.Event()
def rx(e):
self.assertRaisesErrno(zmq.ENOTSUP, receiver.recv)
e.send()
spawn(rx, done1)
spawn(rx, done2)
sleep()
receiver.close()
done1.wait()
done2.wait()
def test_two_simultaneous_connections(self):
# timing-sensitive test, disabled until we come up with a better
# way to do this
self.pool = self.create_pool(2)
conn = self.pool.get()
self.set_up_dummy_table(conn)
self.fill_up_table(conn)
curs = conn.cursor()
conn2 = self.pool.get()
self.set_up_dummy_table(conn2)
self.fill_up_table(conn2)
curs2 = conn2.cursor()
results = []
LONG_QUERY = "select * from test_table"
SHORT_QUERY = "select * from test_table where row_id <= 20"
evt = event.Event()
def long_running_query():
self.assert_cursor_works(curs)
curs.execute(LONG_QUERY)
results.append(1)
evt.send()
evt2 = event.Event()
def short_running_query():
self.assert_cursor_works(curs2)
curs2.execute(SHORT_QUERY)
results.append(2)
evt2.send()
evy.spawn(long_running_query)
evy.spawn(short_running_query)
evt.wait()
evt2.wait()
results.sort()
self.assertEqual([1, 2], results)
def test_recv_something(self):
DATLEN = 5
listener = sockets.GreenSocket()
listener.bind(('', 0))
listener.listen(50)
address, port = listener.getsockname()
self.assertNotEquals(address, 0)
accepting = event.Event()
sent = event.Event()
received = event.Event()
def server():
# accept the connection in another greenlet
accepting.send()
sock, addr = listener.accept()
s = '1234567890'
sock.send(s)
sent.send(s)
gt_server = spawn(server)
def client():
client = sockets.GreenSocket()
accepting.wait()
sleep(0.5)
client.connect(('127.0.0.1', port))
received_data = client.recv(DATLEN)
received.send(received_data)
gt_client = spawn(client)
sent_data = sent.wait()
received_data = received.wait()
self.assertEquals(sent_data[:DATLEN], received_data)
def test_resize_to_Unlimited(self):
q = Queue(0)
def sender(evt, q):
q.put('hi')
evt.send('done')
evt = event.Event()
gt = spawn(sender, evt, q)
sleep()
self.assertFalse(evt.ready())
q.resize(None)
sleep()
self.assertTrue(evt.ready())
gt.wait()
def test_resize_up(self):
q = Queue(0)
def sender(evt, q):
q.put('hi')
evt.send('done')
evt = event.Event()
gt = spawn(sender, evt, q)
sleep(0)
self.assert_(not evt.ready())
q.resize(1)
sleep(0)
self.assert_(evt.ready())
gt.wait()
def test_block(self):
e = zmq._BlockedThread()
done = event.Event()
self.assertFalse(e)
def block():
e.block()
done.send(1)
spawn(block)
sleep()
self.assertFalse(done.has_result())
e.wake()
done.wait()
def test_recv_spawned_before_send_is_non_blocking(self):
req, rep, port = self.create_bound_pair(zmq.PAIR, zmq.PAIR)
# req.connect(ipc)
# rep.bind(ipc)
sleep()
msg = dict(res=None)
done = event.Event()
def rx():
msg['res'] = rep.recv()
done.send('done')
spawn(rx)
req.send('test')
done.wait()
self.assertEqual(msg['res'], 'test')
def spawn_n (self, function, *args, **kwargs):
"""Create a greenthread to run the *function*, the same as
:meth:`spawn`. The difference is that :meth:`spawn_n` returns
None; the results of *function* are not retrievable.
"""
# if reentering an empty pool, don't try to wait on a coroutine freeing
# itself -- instead, just execute in the current coroutine
current = greenthread.getcurrent()
if self.sem.locked() and current in self.coroutines_running:
self._spawn_n_impl(function, args, kwargs, None)
else:
self.sem.acquire()
g = greenthread.spawn_n(self._spawn_n_impl,
function, args, kwargs, True)
if not self.coroutines_running:
self.no_coros_running = event.Event()
self.coroutines_running.add(g)
def execute(meth, *args, **kwargs):
"""
Execute *meth* in a Python thread, blocking the current coroutine/
greenthread until the method completes.
The primary use case for this is to wrap an object or module that is not
amenable to monkeypatching or any of the other tricks that Eventlet uses
to achieve cooperative yielding. With tpool, you can force such objects to
cooperate with green threads by sticking them in native threads, at the cost
of some overhead.
"""
setup()
# if already in tpool, don't recurse into the tpool
# also, call functions directly if we're inside an import lock, because
# if meth does any importing (sadly common), it will hang
my_thread = threading.currentThread()
if my_thread in _threads or imp.lock_held() or _nthreads == 0:
return meth(*args, **kwargs)
cur = greenthread.getcurrent()
# a mini mixing function to make up for the fact that hash(greenlet) doesn't
# have much variability in the lower bits
k = hash(cur)
k = k + 0x2c865fd + (k >> 5)
k = k ^ 0xc84d1b7 ^ (k >> 7)
thread_index = k % _nthreads
reqq, _thread = _threads[thread_index]
e = event.Event()
reqq.put((e, meth, args, kwargs))
rv = e.wait()
if isinstance(rv, tuple)\
and len(rv) == 3\
and isinstance(rv[1], EXC_CLASSES):
import traceback
(c, e, tb) = rv
if not QUIET:
traceback.print_exception(c, e, tb)
traceback.print_stack()
raise c, e, tb
return rv
def test_zero_max_size(self):
q = Queue(0)
def sender(evt, q):
q.put('hi')
evt.send('done')
def receiver(q):
x = q.get()
return x
evt = event.Event()
gt = spawn(sender, evt, q)
sleep(0)
self.assert_(not evt.ready())
gt2 = spawn(receiver, q)
self.assertEquals(gt2.wait(), 'hi')
self.assertEquals(evt.wait(), 'done')
gt.wait()
def test_yields(self):
conn = self.connection
self.set_up_dummy_table(conn)
self.fill_up_table(conn)
curs = conn.cursor()
results = []
SHORT_QUERY = "select * from test_table"
evt = event.Event()
def a_query():
self.assert_cursor_works(curs)
curs.execute(SHORT_QUERY)
results.append(2)
evt.send()
spawn(a_query)
results.append(1)
self.assertEqual([1], results)
evt.wait()
self.assertEqual([1, 2], results)
def test_concurrency(self):
evt = event.Event()
def waiter(sock, addr):
sock.sendall(s2b('hi'))
evt.wait()
l = convenience.listen(('localhost', 0))
gt = spawn(convenience.serve, l, waiter, 5)
def test_client():
c = convenience.connect(('localhost', l.getsockname()[1]))
# verify the client is connected by getting data
self.assertEquals(s2b('hi'), c.recv(2))
return c
clients = [test_client() for i in xrange(5)]
# very next client should not get anything
x = with_timeout(0.01, test_client, timeout_value="timed out")
self.assertEquals(x, "timed out")
def test_sendall_impl(many_bytes):
bufsize = max(many_bytes // 15, 2)
received = event.Event()
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind(("", 0))
listener.listen(50)
_, listener_port = listener.getsockname()
def sender(listener):
(sock, addr) = listener.accept()
sock = bufsized(sock, size=bufsize)
sock.sendall(s2b('x') * many_bytes)
sock.sendall(s2b('y') * second_bytes)
received.wait()
def receiver():
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('127.0.0.1', listener_port))
total = 0
while total < many_bytes:
data = client.recv(
min(many_bytes - total, many_bytes // 10))
if not data:
break
total += len(data)
total2 = 0
while total < second_bytes:
data = client.recv(second_bytes)
if not data:
break
total2 += len(data)
received.send()
waitall(spawn(sender, listener), spawn(receiver))
def test_recv_during_send(self):
sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ)
sleep()
num_recvs = 30
done = event.Event()
sender.setsockopt(zmq.HWM, 10)
sender.setsockopt(zmq.SNDBUF, 10)
receiver.setsockopt(zmq.RCVBUF, 10)
def tx():
tx_i = 0
while tx_i <= 1000:
sender.send(str(tx_i))
tx_i += 1
done.send(0)
spawn(tx)
final_i = done.wait()
self.assertEqual(final_i, 0)
def test_timeout_and_final_write(self):
"""
This test verifies that a write on a socket that we've stopped listening for doesn't
result in an incorrect switch
"""
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(('127.0.0.1', 0))
server.listen(50)
_, bound_port = server.getsockname()
self.assertNotEqual(bound_port, 0)
def sender(evt):
s2, addr = server.accept()
wrap_wfile = s2.makefile('w')
sleep(0.02)
wrap_wfile.write('hi')
s2.close()
evt.send('sent via event')
from evy import event
evt = event.Event()
spawn(sender, evt)
sleep(0) # lets the socket enter accept mode, which
# is necessary for connect to succeed on windows
try:
# try and get some data off of this pipe
# but bail before any is sent
Timeout(0.01)
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(('127.0.0.1', bound_port))
wrap_rfile = client.makefile()
_c = wrap_rfile.read(1)
self.fail()
except socket.error, e:
self.fail('could not connect to port %d: %s' %
(bound_port, str(e)))
def test_returns_immediately(self):
self.pool = self.create_pool()
conn = self.pool.get()
self.set_up_dummy_table(conn)
self.fill_up_table(conn)
curs = conn.cursor()
results = []
SHORT_QUERY = "select * from test_table"
evt = event.Event()
def a_query():
self.assert_cursor_works(curs)
curs.execute(SHORT_QUERY)
results.append(2)
evt.send()
evy.spawn(a_query)
results.append(1)
self.assertEqual([1], results)
evt.wait()
self.assertEqual([1, 2], results)
self.pool.put(conn)
def test_send_1k_push_pull(self):
down, up, port = self.create_bound_pair(zmq.PUSH, zmq.PULL)
sleep()
done = event.Event()
def tx():
tx_i = 0
while tx_i <= 1000:
tx_i += 1
down.send(str(tx_i))
def rx():
while True:
rx_i = up.recv()
if rx_i == "1000":
done.send(0)
break
spawn(tx)
spawn(rx)
final_i = done.wait()
self.assertEqual(final_i, 0)
