def test_waiting(self): pool = GreenPool(1) done = Event() def consume(): done.wait() def waiter(pool): evt = pool.spawn(consume) evt.wait() waiters = [] waiters.append(spawn(waiter, pool)) sleep(0) self.assertEqual(pool.waiting(), 0) waiters.append(spawn(waiter, pool)) sleep(0) self.assertEqual(pool.waiting(), 1) waiters.append(spawn(waiter, pool)) sleep(0) self.assertEqual(pool.waiting(), 2) done.send(None) for w in waiters: w.wait() self.assertEqual(pool.waiting(), 0)
def test_send_many(self): listener = convenience.listen(('', 0)) _, listener_port = listener.getsockname() def server(): # accept the connection in another greenlet sock, addr = listener.accept() total_recv = 0 while True: data = sock.recv(1000) if not data: break total_recv += len(data) return total_recv def client(): client = sockets.GreenSocket() client.connect(('127.0.0.1', listener_port)) msg = s2b("A") * ( 10000) # large enough number to overwhelm most buffers # want to exceed the size of the OS buffer so it'll block in a single send total_sent = 0 for x in range(10): total_sent += client.send(msg) return total_sent res = waitall(spawn(client), spawn(server)) self.assertEqual(res[0], res[1])
def test_send_something (self): listener = convenience.listen(('', 0)) _, listener_port = listener.getsockname() def server (): # accept the connection in another greenlet sock, addr = listener.accept() data = '' while True: last_data = sock.recv(1000) if not last_data: break data += last_data return data def client (): client = sockets.GreenSocket() client.connect(('127.0.0.1', listener_port)) msg = 'hhheeeeelloooooo' total_sent = 0 client.send(msg) return msg res = waitall(spawn(client), spawn(server)) self.assertEquals(res[0], res[1])
def test_nested_acquire(self): q = zmq._QueueLock() self.assertFalse(q) q.acquire() q.acquire() s = semaphore.Semaphore(0) results = [] def lock(x): with q: results.append(x) s.release() spawn(lock, 1) sleep() self.assertEquals(results, []) q.release() sleep() self.assertEquals(results, []) self.assertTrue(q) q.release() s.acquire() self.assertEquals(results, [1])
def test_nested_acquire (self): q = zmq._QueueLock() self.assertFalse(q) q.acquire() q.acquire() s = semaphore.Semaphore(0) results = [] def lock (x): with q: results.append(x) s.release() spawn(lock, 1) sleep() self.assertEquals(results, []) q.release() sleep() self.assertEquals(results, []) self.assertTrue(q) q.release() s.acquire() self.assertEquals(results, [1])
def test_ssl_connect2 (self): def accept_once (listenfd): try: conn, addr = listenfd.accept() conn.write('hello\r\n') shutdown_safe(conn) conn.close() finally: shutdown_safe(listenfd) listenfd.close() server = ssl_listener(('0.0.0.0', 0), self.certificate_file, self.private_key_file) greenthread.spawn(accept_once, server) raw_client = connect(('127.0.0.1', server.getsockname()[1])) client = util.wrap_ssl(raw_client) fd = socket._fileobject(client, 'rb', 8192) assert fd.readline() == 'hello\r\n' try: self.assertEquals('', fd.read(10)) except SSL.ZeroReturnError: # if it's a GreenSSL object it'll do this pass shutdown_safe(client) client.close()
def test_send_1k_req_rep(self): req, rep, port = self.create_bound_pair(zmq.REQ, zmq.REP) sleep() done = event.Event() def tx(): tx_i = 0 req.send(str(tx_i)) while req.recv() != 'done': tx_i += 1 req.send(str(tx_i)) done.send(0) def rx(): while True: rx_i = rep.recv() if rx_i == "1000": rep.send('done') break rep.send('i') spawn(tx) spawn(rx) final_i = done.wait() self.assertEqual(final_i, 0)
def test_recv_into(self): self.reset_timeout(100000) listener = sockets.GreenSocket() listener.bind(('', 0)) listener.listen(50) address, port = listener.getsockname() self.assertNotEquals(address, 0) accepting = event.Event() received = event.Event() sent_data = '1234567890' def server(): # accept the connection in another greenlet accepting.send() sock, addr = listener.accept() sock.send(sent_data) def client(): buf = buffer(array.array('B')) client = sockets.GreenSocket() accepting.wait() sleep(0.5) client.connect(('127.0.0.1', port)) client.recv_into(buf, 5000) received.send(buf) waitall(spawn(client), spawn(server)) received_data = received.wait() self.assertEquals(sent_data, received_data)
def test_send_something(self): listener = convenience.listen(('', 0)) _, listener_port = listener.getsockname() def server(): # accept the connection in another greenlet sock, addr = listener.accept() data = '' while True: last_data = sock.recv(1000) if not last_data: break data += last_data return data def client(): client = sockets.GreenSocket() client.connect(('127.0.0.1', listener_port)) msg = 'hhheeeeelloooooo' total_sent = 0 client.send(msg) return msg res = waitall(spawn(client), spawn(server)) self.assertEquals(res[0], res[1])
def test_send_1k_req_rep (self): req, rep, port = self.create_bound_pair(zmq.REQ, zmq.REP) sleep() done = event.Event() def tx (): tx_i = 0 req.send(str(tx_i)) while req.recv() != 'done': tx_i += 1 req.send(str(tx_i)) done.send(0) def rx (): while True: rx_i = rep.recv() if rx_i == "1000": rep.send('done') break rep.send('i') spawn(tx) spawn(rx) final_i = done.wait() self.assertEqual(final_i, 0)
def test_recv_into (self): self.reset_timeout(100000) listener = sockets.GreenSocket() listener.bind(('', 0)) listener.listen(50) address, port = listener.getsockname() self.assertNotEquals(address, 0) accepting = event.Event() received = event.Event() sent_data = '1234567890' def server (): # accept the connection in another greenlet accepting.send() sock, addr = listener.accept() sock.send(sent_data) def client (): buf = buffer(array.array('B')) client = sockets.GreenSocket() accepting.wait() sleep(0.5) client.connect(('127.0.0.1', port)) client.recv_into(buf, 5000) received.send(buf) waitall(spawn(client), spawn(server)) received_data = received.wait() self.assertEquals(sent_data, received_data)
def test_send_many (self): listener = convenience.listen(('', 0)) _, listener_port = listener.getsockname() def server (): # accept the connection in another greenlet sock, addr = listener.accept() total_recv = 0 while True: data = sock.recv(1000) if not data: break total_recv += len(data) return total_recv def client (): client = sockets.GreenSocket() client.connect(('127.0.0.1', listener_port)) msg = s2b("A") * (10000) # large enough number to overwhelm most buffers # want to exceed the size of the OS buffer so it'll block in a single send total_sent = 0 for x in range(10): total_sent += client.send(msg) return total_sent res = waitall(spawn(client), spawn(server)) self.assertEqual(res[0], res[1])
def test_timer_cancelled_upon_greenlet_exit (self): def func (): spawn_after_local(0.1, self.lst.pop) spawn(func) assert self.lst == [1], self.lst sleep(0.2) assert self.lst == [1], self.lst
def test_senders_that_die(self): q = Queue() def do_send(q): q.put('sent') spawn(do_send, q) self.assertEquals(q.join(), 'sent')
def test_senders_that_die (self): q = Queue() def do_send (q): q.put('sent') spawn(do_send, q) self.assertEquals(q.join(), 'sent')
def test_spawn_is_not_cancelled (self): def func (): spawn(self.lst.pop) # exiting immediatelly, but self.lst.pop must be called spawn(func) sleep(0.1) assert self.lst == [], self.lst
def test_spawn_is_not_cancelled(self): def func(): spawn(self.lst.pop) # exiting immediatelly, but self.lst.pop must be called spawn(func) sleep(0.1) assert self.lst == [], self.lst
def test_timer_cancelled_upon_greenlet_exit(self): def func(): spawn_after_local(0.1, self.lst.pop) spawn(func) assert self.lst == [1], self.lst sleep(0.2) assert self.lst == [1], self.lst
def test_timer_fired (self): def func (): spawn_after_local(0.1, self.lst.pop) sleep(0.2) spawn(func) assert self.lst == [1], self.lst sleep(0.3) assert self.lst == [], self.lst
def test_two_bogus_waiters (self): q = Queue() gt1 = spawn(do_bail, q) gt2 = spawn(do_bail, q) sleep(0) q.put('sent') self.assertEquals(gt1.wait(), 'timed out') self.assertEquals(gt2.wait(), 'timed out') self.assertEquals(q.get(), 'sent')
def test_evy(): event1 = event.Event() event2 = event.Event() event1.send() thread1 = threads.spawn(run, event1, event2) thread2 = threads.spawn(run, event2, event1) thread1.wait() thread2.wait()
def test_timer_fired(self): def func(): spawn_after_local(0.1, self.lst.pop) sleep(0.2) spawn(func) assert self.lst == [1], self.lst sleep(0.3) assert self.lst == [], self.lst
def test_close_with_makefile (self): def accept_close_early (listener): # verify that the makefile and the socket are truly independent # by closing the socket prior to using the made file try: conn, addr = listener.accept() fd = conn.makefile('w') conn.close() fd.write('hello\n') fd.close() self.assertWriteToClosedFileRaises(fd) self.assertRaises(socket.error, conn.send, s2b('b')) finally: listener.close() def accept_close_late (listener): # verify that the makefile and the socket are truly independent # by closing the made file and then sending a character try: conn, addr = listener.accept() fd = conn.makefile('w') fd.write('hello') fd.close() conn.send(s2b('\n')) conn.close() self.assertWriteToClosedFileRaises(fd) self.assertRaises(socket.error, conn.send, s2b('b')) finally: listener.close() def did_it_work (server): client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) _, port = server.getsockname() print 'connecting to port', port client.connect(('127.0.0.1', port)) fd = client.makefile() client.close() assert fd.readline() == 'hello\n' assert fd.read() == '' fd.close() server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.bind(('0.0.0.0', 61222)) self.assertEqual(server.getsockname()[1], 61222) server.listen(50) killer = spawn(accept_close_early, server) did_it_work(server) killer.wait() server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.bind(('0.0.0.0', 0)) server.listen(50) killer = spawn(accept_close_late, server) did_it_work(server) killer.wait()
def test_evy (): event1 = event.Event() event2 = event.Event() event1.send() thread1 = threads.spawn(run, event1, event2) thread2 = threads.spawn(run, event2, event1) thread1.wait() thread2.wait()
def test_two_bogus_waiters(self): q = Queue() gt1 = spawn(do_bail, q) gt2 = spawn(do_bail, q) sleep(0) q.put('sent') self.assertEquals(gt1.wait(), 'timed out') self.assertEquals(gt2.wait(), 'timed out') self.assertEquals(q.get(), 'sent')
def test_two_waiters_one_dies(self): def waiter(q): return q.get() q = Queue() dying = spawn(do_bail, q) waiting = spawn(waiter, q) sleep(0) q.put('hi') self.assertEquals(dying.wait(), 'timed out') self.assertEquals(waiting.wait(), 'hi')
def test_two_waiters_one_dies (self): def waiter (q): return q.get() q = Queue() dying = spawn(do_bail, q) waiting = spawn(waiter, q) sleep(0) q.put('hi') self.assertEquals(dying.wait(), 'timed out') self.assertEquals(waiting.wait(), 'hi')
def test_send (self): event1 = Event() event2 = Event() spawn(event1.send, 'hello event1') Timeout(0, ValueError('interrupted')) try: result = event1.wait() except ValueError: X = object() result = with_timeout(DELAY, event2.wait, timeout_value = X) assert result is X, 'Nobody sent anything to event2 yet it received %r' % (result, )
def test_send_last(self): q = Queue() def waiter(q): timer = Timeout(0.2) self.assertEquals(q.join(), 'hi2') timer.cancel() spawn(waiter, q) sleep(0) sleep(0) q.put('hi2')
def test_send_last (self): q = Queue() def waiter (q): timer = Timeout(0.2) self.assertEquals(q.join(), 'hi2') timer.cancel() spawn(waiter, q) sleep(0) sleep(0) q.put('hi2')
def test_constructing_from_pool(self): pool = GreenPool(2) pile1 = GreenPile(pool) pile2 = GreenPile(pool) def bunch_of_work(pile, unique): for i in xrange(10): pile.spawn(passthru, i + unique) spawn(bunch_of_work, pile1, 0) spawn(bunch_of_work, pile2, 100) sleep(0) self.assertEquals(list(pile2), list(xrange(100, 110))) self.assertEquals(list(pile1), list(xrange(10)))
def test_wrap_ssl (self): server = convenience.wrap_ssl(convenience.listen(('localhost', 0)), certfile = certificate_file, keyfile = private_key_file, server_side = True) port = server.getsockname()[1] def handle (sock, addr): sock.sendall(sock.recv(1024)) raise convenience.StopServe() spawn(convenience.serve, server, handle) client = convenience.wrap_ssl(convenience.connect(('localhost', port))) client.sendall("echo") self.assertEquals("echo", client.recv(1024))
def test_constructing_from_pool (self): pool = GreenPool(2) pile1 = GreenPile(pool) pile2 = GreenPile(pool) def bunch_of_work (pile, unique): for i in xrange(10): pile.spawn(passthru, i + unique) spawn(bunch_of_work, pile1, 0) spawn(bunch_of_work, pile2, 100) sleep(0) self.assertEquals(list(pile2), list(xrange(100, 110))) self.assertEquals(list(pile1), list(xrange(10)))
def test_bounded (self): sem = semaphore.CappedSemaphore(2, limit = 3) self.assertEqual(sem.acquire(), True) self.assertEqual(sem.acquire(), True) gt1 = spawn(sem.release) self.assertEqual(sem.acquire(), True) self.assertEqual(-3, sem.balance) sem.release() sem.release() sem.release() gt2 = spawn(sem.acquire) sem.release() self.assertEqual(3, sem.balance) gt1.wait() gt2.wait()
def test_waiting (self): def do_wait (q, evt): result = q.join() evt.send(result) q = Queue() e1 = Event() spawn(do_wait, q, e1) sleep(0) self.assertEquals(1, q.join()) q.put('hi') sleep(0) self.assertEquals(0, q.join()) self.assertEquals('hi', e1.wait()) self.assertEquals(0, q.join())
def test_assignment(self): my_local = corolocal.local() my_local.a = 1 def do_something(): my_local.b = 2 self.assertEqual(my_local.b, 2) try: my_local.a self.fail() except AttributeError: pass spawn(do_something).wait() self.assertEqual(my_local.a, 1)
def test_bounded(self): sem = semaphore.CappedSemaphore(2, limit=3) self.assertEqual(sem.acquire(), True) self.assertEqual(sem.acquire(), True) gt1 = spawn(sem.release) self.assertEqual(sem.acquire(), True) self.assertEqual(-3, sem.balance) sem.release() sem.release() sem.release() gt2 = spawn(sem.acquire) sem.release() self.assertEqual(3, sem.balance) gt1.wait() gt2.wait()
def test_block (self): e = zmq._BlockedThread() done = event.Event() self.assertFalse(e) def block (): e.block() done.send(1) spawn(block) sleep() self.assertFalse(done.has_result()) e.wake() done.wait()
def test_waiting(self): def do_wait(q, evt): result = q.join() evt.send(result) q = Queue() e1 = Event() spawn(do_wait, q, e1) sleep(0) self.assertEquals(1, q.join()) q.put('hi') sleep(0) self.assertEquals(0, q.join()) self.assertEquals('hi', e1.wait()) self.assertEquals(0, q.join())
def test_wrap_ssl(self): server = convenience.wrap_ssl(convenience.listen(('localhost', 0)), certfile=certificate_file, keyfile=private_key_file, server_side=True) port = server.getsockname()[1] def handle(sock, addr): sock.sendall(sock.recv(1024)) raise convenience.StopServe() spawn(convenience.serve, server, handle) client = convenience.wrap_ssl(convenience.connect(('localhost', port))) client.sendall("echo") self.assertEquals("echo", client.recv(1024))
def test_assignment (self): my_local = corolocal.local() my_local.a = 1 def do_something (): my_local.b = 2 self.assertEqual(my_local.b, 2) try: my_local.a self.fail() except AttributeError: pass spawn(do_something).wait() self.assertEqual(my_local.a, 1)
def test_block(self): e = zmq._BlockedThread() done = event.Event() self.assertFalse(e) def block(): e.block() done.send(1) spawn(block) sleep() self.assertFalse(done.has_result()) e.wake() done.wait()
def test_recv_spawned_before_send_is_non_blocking(self): req, rep, port = self.create_bound_pair(zmq.PAIR, zmq.PAIR) # req.connect(ipc) # rep.bind(ipc) sleep() msg = dict(res=None) done = event.Event() def rx(): msg['res'] = rep.recv() done.send('done') spawn(rx) req.send('test') done.wait() self.assertEqual(msg['res'], 'test')
def test_blocks_on_pool (self): waiter = Queue(0) def greedy (): self.pool.get() self.pool.get() self.pool.get() self.pool.get() # No one should be waiting yet. self.assertEquals(self.pool.waiting(), 0) # The call to the next get will unschedule this routine. self.pool.get() # So this put should never be called. waiter.put('Failed!') killable = spawn(greedy) # no one should be waiting yet. self.assertEquals(self.pool.waiting(), 0) ## Wait for greedy sleep(0) ## Greedy should be blocking on the last get self.assertEquals(self.pool.waiting(), 1) ## Send will never be called, so balance should be 0. self.assertFalse(not waiter.full()) kill(killable)
def test_sendall_timeout(self): listener = sockets.GreenSocket() listener.bind(('', 0)) listener.listen(50) evt = event.Event() def server(): # accept the connection in another greenlet sock, addr = listener.accept() evt.wait() gt = spawn(server) addr = listener.getsockname() self.assertNotEqual(addr[1], 0) client = sockets.GreenSocket() client.settimeout(0.1) client.connect(addr) try: msg = s2b("A") * (8 * 1024 * 1024) # want to exceed the size of the OS buffer so it'll block client.sendall(msg) self.fail("socket.timeout not raised") except socket.timeout, e: self.assert_(hasattr(e, 'args')) self.assertEqual(e.args[1], 'timed out')
def test_recv_into_timeout (self): buf = buffer(array.array('B')) listener = sockets.GreenSocket() listener.bind(('', 0)) listener.listen(50) address, port = listener.getsockname() self.assertNotEquals(address, 0) accepting = event.Event() accepted = event.Event() def server (): # accept the connection in another greenlet accepting.send() sock, addr = listener.accept() accepted.wait() gt = spawn(server) client = sockets.GreenSocket() client.settimeout(0.1) accepting.wait() client.connect(('127.0.0.1', port)) try: client.recv_into(buf, 100) self.fail("socket.timeout not raised") except socket.timeout, e: self.assert_(hasattr(e, 'args')) self.assertEqual(e.args[0], 'timed out')
def test_blocks_on_pool(self): waiter = Queue(0) def greedy(): self.pool.get() self.pool.get() self.pool.get() self.pool.get() # No one should be waiting yet. self.assertEquals(self.pool.waiting(), 0) # The call to the next get will unschedule this routine. self.pool.get() # So this put should never be called. waiter.put('Failed!') killable = spawn(greedy) # no one should be waiting yet. self.assertEquals(self.pool.waiting(), 0) ## Wait for greedy sleep(0) ## Greedy should be blocking on the last get self.assertEquals(self.pool.waiting(), 1) ## Send will never be called, so balance should be 0. self.assertFalse(not waiter.full()) kill(killable)
def spawn (self, function, *args, **kwargs): """ Run the *function* with its arguments in its own green thread. Returns the :class:`GreenThread <evy.greenthread.GreenThread>` object that is running the function, which can be used to retrieve the results. If the pool is currently at capacity, ``spawn`` will block until one of the running greenthreads completes its task and frees up a slot. This function is reentrant; *function* can call ``spawn`` on the same pool without risk of deadlocking the whole thing. """ # if reentering an empty pool, don't try to wait on a coroutine freeing # itself -- instead, just execute in the current coroutine current = greenthread.getcurrent() if self.sem.locked() and current in self.coroutines_running: # a bit hacky to use the GT without switching to it gt = greenthread.GreenThread(current) gt.main(function, args, kwargs) return gt else: self.sem.acquire() gt = greenthread.spawn(function, *args, **kwargs) if not self.coroutines_running: self.no_coros_running = event.Event() self.coroutines_running.add(gt) gt.link(self._spawn_done) return gt
def test_pipe_read (self): # ensure that 'readline' works properly on GreenPipes when data is not # immediately available (fd is nonblocking, was raising EAGAIN) # also ensures that readline() terminates on '\n' and '\r\n' r, w = os.pipe() r = GreenPipe(r) w = GreenPipe(w, 'w') def writer (): sleep(.1) w.write('line\n') w.flush() w.write('line\r\n') w.flush() gt = spawn(writer) sleep(0) line = r.readline() self.assertEquals(line, 'line\n') line = r.readline() self.assertEquals(line, 'line\r\n') gt.wait()
def test_recv_into_timeout(self): buf = buffer(array.array('B')) listener = sockets.GreenSocket() listener.bind(('', 0)) listener.listen(50) address, port = listener.getsockname() self.assertNotEquals(address, 0) accepting = event.Event() accepted = event.Event() def server(): # accept the connection in another greenlet accepting.send() sock, addr = listener.accept() accepted.wait() gt = spawn(server) client = sockets.GreenSocket() client.settimeout(0.1) accepting.wait() client.connect(('127.0.0.1', port)) try: client.recv_into(buf, 100) self.fail("socket.timeout not raised") except socket.timeout, e: self.assert_(hasattr(e, 'args')) self.assertEqual(e.args[0], 'timed out')
def test_channel_wait(self): channel = Queue(0) events = [] def another_greenlet(): events.append('sending hello') channel.put('hello') events.append('sending world') channel.put('world') events.append('sent world') gt = spawn(another_greenlet) events.append('waiting') events.append(channel.get()) events.append(channel.get()) self.assertEqual( ['waiting', 'sending hello', 'hello', 'sending world', 'world'], events) sleep(0) self.assertEqual([ 'waiting', 'sending hello', 'hello', 'sending world', 'world', 'sent world' ], events)
def test_recv_spawned_before_send_is_non_blocking (self): req, rep, port = self.create_bound_pair(zmq.PAIR, zmq.PAIR) # req.connect(ipc) # rep.bind(ipc) sleep() msg = dict(res = None) done = event.Event() def rx (): msg['res'] = rep.recv() done.send('done') spawn(rx) req.send('test') done.wait() self.assertEqual(msg['res'], 'test')
def test_del_closes_socket (self): def accept_once (listener): # delete/overwrite the original conn # object, only keeping the file object around # closing the file object should close everything try: conn, addr = listener.accept() conn = conn.makefile('w') conn.write('hello\n') conn.close() self.assertWriteToClosedFileRaises(conn) finally: listener.close() server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.bind(('127.0.0.1', 0)) server.listen(50) killer = spawn(accept_once, server) client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client.connect(('127.0.0.1', server.getsockname()[1])) fd = client.makefile() client.close() assert fd.read() == 'hello\n' assert fd.read() == '' killer.wait()
def test_wrap_iterator2 (self): self.reset_timeout(5) # might take a while due to imprecise sleeping def foo (): import time for x in xrange(2): yield x time.sleep(0.001) counter = [0] def tick (): for i in xrange(20000): counter[0] += 1 if counter[0] % 20 == 0: sleep(0.0001) else: sleep() gt = spawn(tick) previtem = 0 for item in tpool.Proxy(foo()): self.assert_(item >= previtem) # make sure the tick happened at least a few times so that we know # that our iterations in foo() were actually tpooled self.assert_(counter[0] > 10, counter[0]) gt.kill()