def test_recv_into_timeout(self): buf = buffer(array.array('B')) listener = sockets.GreenSocket() listener.bind(('', 0)) listener.listen(50) address, port = listener.getsockname() self.assertNotEquals(address, 0) accepting = event.Event() accepted = event.Event() def server(): # accept the connection in another greenlet accepting.send() sock, addr = listener.accept() accepted.wait() gt = spawn(server) client = sockets.GreenSocket() client.settimeout(0.1) accepting.wait() client.connect(('127.0.0.1', port)) try: client.recv_into(buf, 100) self.fail("socket.timeout not raised") except socket.timeout, e: self.assert_(hasattr(e, 'args')) self.assertEqual(e.args[0], 'timed out')
def test_recv_into(self): self.reset_timeout(100000) listener = sockets.GreenSocket() listener.bind(('', 0)) listener.listen(50) address, port = listener.getsockname() self.assertNotEquals(address, 0) accepting = event.Event() received = event.Event() sent_data = '1234567890' def server(): # accept the connection in another greenlet accepting.send() sock, addr = listener.accept() sock.send(sent_data) def client(): buf = buffer(array.array('B')) client = sockets.GreenSocket() accepting.wait() sleep(0.5) client.connect(('127.0.0.1', port)) client.recv_into(buf, 5000) received.send(buf) waitall(spawn(client), spawn(server)) received_data = received.wait() self.assertEquals(sent_data, received_data)
def test_evy(): event1 = event.Event() event2 = event.Event() event1.send() thread1 = threads.spawn(run, event1, event2) thread2 = threads.spawn(run, event2, event1) thread1.wait() thread2.wait()
def test_send_1k_req_rep(self): req, rep, port = self.create_bound_pair(zmq.REQ, zmq.REP) sleep() done = event.Event() def tx(): tx_i = 0 req.send(str(tx_i)) while req.recv() != 'done': tx_i += 1 req.send(str(tx_i)) done.send(0) def rx(): while True: rx_i = rep.recv() if rx_i == "1000": rep.send('done') break rep.send('i') spawn(tx) spawn(rx) final_i = done.wait() self.assertEqual(final_i, 0)
def test_waiters_get_woken(self): # verify that when there's someone waiting on an empty pool # and someone puts an immediately-closed connection back in # the pool that the waiter gets woken self.pool.put(self.connection) self.pool.clear() self.pool = self.create_pool(max_size=1, max_age=0) self.connection = self.pool.get() self.assertEquals(self.pool.free(), 0) self.assertEquals(self.pool.waiting(), 0) e = event.Event() def retrieve(pool, ev): c = pool.get() ev.send(c) evy.spawn(retrieve, self.pool, e) sleep(0) # these two sleeps should advance the retrieve sleep(0) # coroutine until it's waiting in get() self.assertEquals(self.pool.free(), 0) self.assertEquals(self.pool.waiting(), 1) self.pool.put(self.connection) timer = Timeout(1) conn = e.wait() timer.cancel() self.assertEquals(self.pool.free(), 0) self.assertEquals(self.pool.waiting(), 0) self.pool.put(conn)
def test_send_during_recv(self): sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) sleep() num_recvs = 30 done_evts = [event.Event() for _ in range(num_recvs)] def slow_rx(done, msg): self.assertEqual(sender.recv(), msg) done.send(0) def tx(): tx_i = 0 while tx_i <= 1000: sender.send(str(tx_i)) tx_i += 1 def rx(): while True: rx_i = receiver.recv() if rx_i == "1000": for i in range(num_recvs): receiver.send('done%d' % i) sleep() return for i in range(num_recvs): spawn(slow_rx, done_evts[i], "done%d" % i) spawn(tx) spawn(rx) for evt in done_evts: self.assertEqual(evt.wait(), 0)
def test_send_during_recv_multipart(self): sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) sleep() num_recvs = 30 done_evts = [event.Event() for _ in range(num_recvs)] def slow_rx(done, msg): self.assertEqual(sender.recv_multipart(), msg) done.send(0) def tx(): tx_i = 0 while tx_i <= 1000: sender.send_multipart([str(tx_i), '1', '2', '3']) tx_i += 1 def rx(): while True: rx_i = receiver.recv_multipart() if rx_i == ["1000", '1', '2', '3']: for i in range(num_recvs): receiver.send_multipart(['done%d' % i, 'a', 'b', 'c']) sleep() return for i in range(num_recvs): spawn(slow_rx, done_evts[i], ["done%d" % i, 'a', 'b', 'c']) spawn(tx) spawn(rx) for i in range(num_recvs): final_i = done_evts[i].wait() self.assertEqual(final_i, 0)
def test_sendall_timeout(self): listener = sockets.GreenSocket() listener.bind(('', 0)) listener.listen(50) evt = event.Event() def server(): # accept the connection in another greenlet sock, addr = listener.accept() evt.wait() gt = spawn(server) addr = listener.getsockname() self.assertNotEqual(addr[1], 0) client = sockets.GreenSocket() client.settimeout(0.1) client.connect(addr) try: msg = s2b("A") * (8 * 1024 * 1024) # want to exceed the size of the OS buffer so it'll block client.sendall(msg) self.fail("socket.timeout not raised") except socket.timeout, e: self.assert_(hasattr(e, 'args')) self.assertEqual(e.args[1], 'timed out')
def test_send_timeout(self): self.reset_timeout(1000000) listener = convenience.listen(('', 0)) _, port = listener.getsockname() evt = event.Event() def server(): # accept the connection in another greenlet sock, addr = listener.accept() evt.wait() def client(): client = sockets.GreenSocket() client.connect(('127.0.0.1', port)) try: client.settimeout(0.00001) msg = s2b("A") * ( 100000) # large enough number to overwhelm most buffers total_sent = 0 # want to exceed the size of the OS buffer so it'll block in a # single send for x in range(10): total_sent += client.send(msg) self.fail("socket.timeout not raised") except socket.timeout, e: self.assert_(hasattr(e, 'args')) self.assertEqual(e.args[1], 'timed out') finally:
def test_app_socket_errors_76 (self): error_detected = [False] done_with_request = event.Event() site = self.site def error_detector (environ, start_response): try: try: return site(environ, start_response) except: error_detected[0] = True raise finally: done_with_request.send(True) self.site = error_detector self.spawn_server() connect_data = [ "GET /error HTTP/1.1", "Upgrade: WebSocket", "Connection: Upgrade", "Host: localhost:%s" % self.port, "Origin: http://localhost:%s" % self.port, "Sec-WebSocket-Protocol: ws", "Sec-WebSocket-Key1: 4 @1 46546xW%0l 1 5", "Sec-WebSocket-Key2: 12998 5 Y3 1 .P00", ] sock = connect( ('localhost', self.port)) sock.sendall('\r\n'.join(connect_data) + '\r\n\r\n^n:ds[4U') resp = sock.recv(1024) done_with_request.wait() self.assert_(error_detected[0])
def spawn (self, function, *args, **kwargs): """ Run the *function* with its arguments in its own green thread. Returns the :class:`GreenThread <evy.greenthread.GreenThread>` object that is running the function, which can be used to retrieve the results. If the pool is currently at capacity, ``spawn`` will block until one of the running greenthreads completes its task and frees up a slot. This function is reentrant; *function* can call ``spawn`` on the same pool without risk of deadlocking the whole thing. """ # if reentering an empty pool, don't try to wait on a coroutine freeing # itself -- instead, just execute in the current coroutine current = greenthread.getcurrent() if self.sem.locked() and current in self.coroutines_running: # a bit hacky to use the GT without switching to it gt = greenthread.GreenThread(current) gt.main(function, args, kwargs) return gt else: self.sem.acquire() gt = greenthread.spawn(function, *args, **kwargs) if not self.coroutines_running: self.no_coros_running = event.Event() self.coroutines_running.add(gt) gt.link(self._spawn_done) return gt
def test_breaking_the_connection_75 (self): error_detected = [False] done_with_request = event.Event() site = self.site def error_detector (environ, start_response): try: try: return site(environ, start_response) except: error_detected[0] = True raise finally: done_with_request.send(True) self.site = error_detector self.spawn_server() connect_data = [ "GET /range HTTP/1.1", "Upgrade: WebSocket", "Connection: Upgrade", "Host: localhost:%s" % self.port, "Origin: http://localhost:%s" % self.port, "WebSocket-Protocol: ws", ] sock = connect( ('localhost', self.port)) sock.sendall('\r\n'.join(connect_data) + '\r\n\r\n') resp = sock.recv(1024) # get the headers sock.close() # close while the app is running done_with_request.wait() self.assert_(not error_detected[0])
def test_send_1k_pub_sub(self): pub, sub_all, port = self.create_bound_pair(zmq.PUB, zmq.SUB) sub1 = self.context.socket(zmq.SUB) sub2 = self.context.socket(zmq.SUB) self.sockets.extend([sub1, sub2]) addr = 'tcp://127.0.0.1:%s' % port sub1.connect(addr) sub2.connect(addr) sub_all.setsockopt(zmq.SUBSCRIBE, '') sub1.setsockopt(zmq.SUBSCRIBE, 'sub1') sub2.setsockopt(zmq.SUBSCRIBE, 'sub2') sub_all_done = event.Event() sub1_done = event.Event() sub2_done = event.Event() sleep(0.2) def rx(sock, done_evt, msg_count=10000): count = 0 while count < msg_count: msg = sock.recv() sleep() if 'LAST' in msg: break count += 1 done_evt.send(count) def tx(sock): for i in range(1, 1001): msg = "sub%s %s" % ([2, 1][i % 2], i) sock.send(msg) sleep() sock.send('sub1 LAST') sock.send('sub2 LAST') spawn(rx, sub_all, sub_all_done) spawn(rx, sub1, sub1_done) spawn(rx, sub2, sub2_done) spawn(tx, pub) sub1_count = sub1_done.wait() sub2_count = sub2_done.wait() sub_all_count = sub_all_done.wait() self.assertEqual(sub1_count, 500) self.assertEqual(sub2_count, 500) self.assertEqual(sub_all_count, 1000)
def test_close_during_recv(self): sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) sleep() done1 = event.Event() done2 = event.Event() def rx(e): self.assertRaisesErrno(zmq.ENOTSUP, receiver.recv) e.send() spawn(rx, done1) spawn(rx, done2) sleep() receiver.close() done1.wait() done2.wait()
def test_two_simultaneous_connections(self): # timing-sensitive test, disabled until we come up with a better # way to do this self.pool = self.create_pool(2) conn = self.pool.get() self.set_up_dummy_table(conn) self.fill_up_table(conn) curs = conn.cursor() conn2 = self.pool.get() self.set_up_dummy_table(conn2) self.fill_up_table(conn2) curs2 = conn2.cursor() results = [] LONG_QUERY = "select * from test_table" SHORT_QUERY = "select * from test_table where row_id <= 20" evt = event.Event() def long_running_query(): self.assert_cursor_works(curs) curs.execute(LONG_QUERY) results.append(1) evt.send() evt2 = event.Event() def short_running_query(): self.assert_cursor_works(curs2) curs2.execute(SHORT_QUERY) results.append(2) evt2.send() evy.spawn(long_running_query) evy.spawn(short_running_query) evt.wait() evt2.wait() results.sort() self.assertEqual([1, 2], results)
def test_recv_something(self): DATLEN = 5 listener = sockets.GreenSocket() listener.bind(('', 0)) listener.listen(50) address, port = listener.getsockname() self.assertNotEquals(address, 0) accepting = event.Event() sent = event.Event() received = event.Event() def server(): # accept the connection in another greenlet accepting.send() sock, addr = listener.accept() s = '1234567890' sock.send(s) sent.send(s) gt_server = spawn(server) def client(): client = sockets.GreenSocket() accepting.wait() sleep(0.5) client.connect(('127.0.0.1', port)) received_data = client.recv(DATLEN) received.send(received_data) gt_client = spawn(client) sent_data = sent.wait() received_data = received.wait() self.assertEquals(sent_data[:DATLEN], received_data)
def test_resize_to_Unlimited(self): q = Queue(0) def sender(evt, q): q.put('hi') evt.send('done') evt = event.Event() gt = spawn(sender, evt, q) sleep() self.assertFalse(evt.ready()) q.resize(None) sleep() self.assertTrue(evt.ready()) gt.wait()
def test_resize_up(self): q = Queue(0) def sender(evt, q): q.put('hi') evt.send('done') evt = event.Event() gt = spawn(sender, evt, q) sleep(0) self.assert_(not evt.ready()) q.resize(1) sleep(0) self.assert_(evt.ready()) gt.wait()
def test_block(self): e = zmq._BlockedThread() done = event.Event() self.assertFalse(e) def block(): e.block() done.send(1) spawn(block) sleep() self.assertFalse(done.has_result()) e.wake() done.wait()
def test_recv_spawned_before_send_is_non_blocking(self): req, rep, port = self.create_bound_pair(zmq.PAIR, zmq.PAIR) # req.connect(ipc) # rep.bind(ipc) sleep() msg = dict(res=None) done = event.Event() def rx(): msg['res'] = rep.recv() done.send('done') spawn(rx) req.send('test') done.wait() self.assertEqual(msg['res'], 'test')
def spawn_n (self, function, *args, **kwargs): """Create a greenthread to run the *function*, the same as :meth:`spawn`. The difference is that :meth:`spawn_n` returns None; the results of *function* are not retrievable. """ # if reentering an empty pool, don't try to wait on a coroutine freeing # itself -- instead, just execute in the current coroutine current = greenthread.getcurrent() if self.sem.locked() and current in self.coroutines_running: self._spawn_n_impl(function, args, kwargs, None) else: self.sem.acquire() g = greenthread.spawn_n(self._spawn_n_impl, function, args, kwargs, True) if not self.coroutines_running: self.no_coros_running = event.Event() self.coroutines_running.add(g)
def execute(meth, *args, **kwargs): """ Execute *meth* in a Python thread, blocking the current coroutine/ greenthread until the method completes. The primary use case for this is to wrap an object or module that is not amenable to monkeypatching or any of the other tricks that Eventlet uses to achieve cooperative yielding. With tpool, you can force such objects to cooperate with green threads by sticking them in native threads, at the cost of some overhead. """ setup() # if already in tpool, don't recurse into the tpool # also, call functions directly if we're inside an import lock, because # if meth does any importing (sadly common), it will hang my_thread = threading.currentThread() if my_thread in _threads or imp.lock_held() or _nthreads == 0: return meth(*args, **kwargs) cur = greenthread.getcurrent() # a mini mixing function to make up for the fact that hash(greenlet) doesn't # have much variability in the lower bits k = hash(cur) k = k + 0x2c865fd + (k >> 5) k = k ^ 0xc84d1b7 ^ (k >> 7) thread_index = k % _nthreads reqq, _thread = _threads[thread_index] e = event.Event() reqq.put((e, meth, args, kwargs)) rv = e.wait() if isinstance(rv, tuple)\ and len(rv) == 3\ and isinstance(rv[1], EXC_CLASSES): import traceback (c, e, tb) = rv if not QUIET: traceback.print_exception(c, e, tb) traceback.print_stack() raise c, e, tb return rv
def test_zero_max_size(self): q = Queue(0) def sender(evt, q): q.put('hi') evt.send('done') def receiver(q): x = q.get() return x evt = event.Event() gt = spawn(sender, evt, q) sleep(0) self.assert_(not evt.ready()) gt2 = spawn(receiver, q) self.assertEquals(gt2.wait(), 'hi') self.assertEquals(evt.wait(), 'done') gt.wait()
def test_yields(self): conn = self.connection self.set_up_dummy_table(conn) self.fill_up_table(conn) curs = conn.cursor() results = [] SHORT_QUERY = "select * from test_table" evt = event.Event() def a_query(): self.assert_cursor_works(curs) curs.execute(SHORT_QUERY) results.append(2) evt.send() spawn(a_query) results.append(1) self.assertEqual([1], results) evt.wait() self.assertEqual([1, 2], results)
def test_concurrency(self): evt = event.Event() def waiter(sock, addr): sock.sendall(s2b('hi')) evt.wait() l = convenience.listen(('localhost', 0)) gt = spawn(convenience.serve, l, waiter, 5) def test_client(): c = convenience.connect(('localhost', l.getsockname()[1])) # verify the client is connected by getting data self.assertEquals(s2b('hi'), c.recv(2)) return c clients = [test_client() for i in xrange(5)] # very next client should not get anything x = with_timeout(0.01, test_client, timeout_value="timed out") self.assertEquals(x, "timed out")
def test_sendall_impl(many_bytes): bufsize = max(many_bytes // 15, 2) received = event.Event() listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listener.bind(("", 0)) listener.listen(50) _, listener_port = listener.getsockname() def sender(listener): (sock, addr) = listener.accept() sock = bufsized(sock, size=bufsize) sock.sendall(s2b('x') * many_bytes) sock.sendall(s2b('y') * second_bytes) received.wait() def receiver(): client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client.connect(('127.0.0.1', listener_port)) total = 0 while total < many_bytes: data = client.recv( min(many_bytes - total, many_bytes // 10)) if not data: break total += len(data) total2 = 0 while total < second_bytes: data = client.recv(second_bytes) if not data: break total2 += len(data) received.send() waitall(spawn(sender, listener), spawn(receiver))
def test_recv_during_send(self): sender, receiver, port = self.create_bound_pair(zmq.XREQ, zmq.XREQ) sleep() num_recvs = 30 done = event.Event() sender.setsockopt(zmq.HWM, 10) sender.setsockopt(zmq.SNDBUF, 10) receiver.setsockopt(zmq.RCVBUF, 10) def tx(): tx_i = 0 while tx_i <= 1000: sender.send(str(tx_i)) tx_i += 1 done.send(0) spawn(tx) final_i = done.wait() self.assertEqual(final_i, 0)
def test_timeout_and_final_write(self): """ This test verifies that a write on a socket that we've stopped listening for doesn't result in an incorrect switch """ server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.bind(('127.0.0.1', 0)) server.listen(50) _, bound_port = server.getsockname() self.assertNotEqual(bound_port, 0) def sender(evt): s2, addr = server.accept() wrap_wfile = s2.makefile('w') sleep(0.02) wrap_wfile.write('hi') s2.close() evt.send('sent via event') from evy import event evt = event.Event() spawn(sender, evt) sleep(0) # lets the socket enter accept mode, which # is necessary for connect to succeed on windows try: # try and get some data off of this pipe # but bail before any is sent Timeout(0.01) client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client.connect(('127.0.0.1', bound_port)) wrap_rfile = client.makefile() _c = wrap_rfile.read(1) self.fail() except socket.error, e: self.fail('could not connect to port %d: %s' % (bound_port, str(e)))
def test_returns_immediately(self): self.pool = self.create_pool() conn = self.pool.get() self.set_up_dummy_table(conn) self.fill_up_table(conn) curs = conn.cursor() results = [] SHORT_QUERY = "select * from test_table" evt = event.Event() def a_query(): self.assert_cursor_works(curs) curs.execute(SHORT_QUERY) results.append(2) evt.send() evy.spawn(a_query) results.append(1) self.assertEqual([1], results) evt.wait() self.assertEqual([1, 2], results) self.pool.put(conn)
def test_send_1k_push_pull(self): down, up, port = self.create_bound_pair(zmq.PUSH, zmq.PULL) sleep() done = event.Event() def tx(): tx_i = 0 while tx_i <= 1000: tx_i += 1 down.send(str(tx_i)) def rx(): while True: rx_i = up.recv() if rx_i == "1000": done.send(0) break spawn(tx) spawn(rx) final_i = done.wait() self.assertEqual(final_i, 0)