def test_multiple_waiters(self):
# tests that multiple waiters get their results back
q = coros.queue()
def waiter(q, evt):
evt.send(q.wait())
sendings = ['1', '2', '3', '4']
evts = [coros.event() for x in sendings]
for i, x in enumerate(sendings):
api.spawn(waiter, q, evts[i])
api.sleep(0.01) # get 'em all waiting
results = set()
def collect_pending_results():
for i, e in enumerate(evts):
timer = api.exc_after(0.001, api.TimeoutError)
try:
x = e.wait()
results.add(x)
timer.cancel()
except api.TimeoutError:
pass # no pending result at that event
return len(results)
q.send(sendings[0])
self.assertEqual(collect_pending_results(), 1)
q.send(sendings[1])
self.assertEqual(collect_pending_results(), 2)
q.send(sendings[2])
q.send(sendings[3])
self.assertEqual(collect_pending_results(), 4)
def fan(self, block, input_list):
queue = coros.queue(0)
results = []
exceptional_results = 0
for index, input_item in enumerate(input_list):
pool_item = self.get()
## Fan out
api.spawn(
self._invoke, block, pool_item, input_item, index, queue)
## Fan back in
for i in range(len(input_list)):
## Wait for all guys to send to the queue
index, value = queue.wait()
if isinstance(value, Exception):
exceptional_results += 1
results.append((index, value))
results.sort()
results = [value for index, value in results]
if exceptional_results:
if exceptional_results == len(results):
raise AllFailed(results)
raise SomeFailed(results)
return results
def fan(self, block, input_list):
queue = coros.queue(0)
results = []
exceptional_results = 0
for index, input_item in enumerate(input_list):
pool_item = self.get()
## Fan out
api.spawn(self._invoke, block, pool_item, input_item, index, queue)
## Fan back in
for i in range(len(input_list)):
## Wait for all guys to send to the queue
index, value = queue.wait()
if isinstance(value, Exception):
exceptional_results += 1
results.append((index, value))
results.sort()
results = [value for index, value in results]
if exceptional_results:
if exceptional_results == len(results):
raise AllFailed(results)
raise SomeFailed(results)
return results
def test_reset(self):
evt = coros.event()
# calling reset before send should throw
self.assertRaises(AssertionError, evt.reset)
value = 'some stuff'
def send_to_event():
evt.send(value)
api.spawn(send_to_event)
self.assertEqual(evt.wait(), value)
# now try it again, and we should get the same exact value,
# and we shouldn't be allowed to resend without resetting
value2 = 'second stuff'
self.assertRaises(AssertionError, evt.send, value2)
self.assertEqual(evt.wait(), value)
# reset and everything should be happy
evt.reset()
def send_to_event2():
evt.send(value2)
api.spawn(send_to_event2)
self.assertEqual(evt.wait(), value2)
def test_timeout_and_final_write(self):
# This test verifies that a write on a socket that we've
# stopped listening for doesn't result in an incorrect switch
rpipe, wpipe = os.pipe()
rfile = os.fdopen(rpipe,"rb",1)
wrap_rfile = greenio.GreenPipe(rfile)
wfile = os.fdopen(wpipe,"wb",1)
wrap_wfile = greenio.GreenPipe(wfile)
def sender(evt):
api.sleep(0.02)
wrap_wfile.write('hi')
evt.send('sent via event')
from eventlib import coros
evt = coros.event()
api.spawn(sender, evt)
try:
# try and get some data off of this pipe
# but bail before any is sent
api.exc_after(0.01, api.TimeoutError)
_c = wrap_rfile.read(1)
self.fail()
except api.TimeoutError:
pass
result = evt.wait()
self.assertEqual(result, 'sent via event')
def test_waiters_get_woken(self):
# verify that when there's someone waiting on an empty pool
# and someone puts an immediately-closed connection back in
# the pool that the waiter gets woken
self.pool = self.create_pool(max_size=1, max_age=0)
conn = self.pool.get()
self.assertEqual(self.pool.free(), 0)
self.assertEqual(self.pool.waiting(), 0)
e = coros.event()
def retrieve(pool, ev):
c = pool.get()
ev.send(c)
api.spawn(retrieve, self.pool, e)
api.sleep(0) # these two sleeps should advance the retrieve
api.sleep(0) # coroutine until it's waiting in get()
self.assertEqual(self.pool.free(), 0)
self.assertEqual(self.pool.waiting(), 1)
self.pool.put(conn)
timer = api.exc_after(0.3, api.TimeoutError)
conn = e.wait()
timer.cancel()
self.assertEqual(self.pool.free(), 0)
self.assertEqual(self.pool.waiting(), 0)
def setup():
global _threads
for i in range(0,_nthreads):
_threads[i] = threading.Thread(target=tworker)
_threads[i].setDaemon(True)
_threads[i].start()
api.spawn(tpool_trampoline)
def test_spawn_is_not_cancelled(self):
def func():
spawn(self.lst.pop)
# exiting immediatelly, but self.lst.pop must be called
spawn(func)
sleep(0.1)
assert self.lst == [], self.lst
def test_timer_cancelled_upon_greenlet_exit(self):
def func():
call_after(0.1, self.lst.pop)
spawn(func)
assert self.lst == [1], self.lst
sleep(0.2)
assert self.lst == [1], self.lst
def setup():
global _threads
for i in range(0, _nthreads):
_threads[i] = threading.Thread(target=tworker)
_threads[i].setDaemon(True)
_threads[i].start()
api.spawn(tpool_trampoline)
def test_senders_that_die(self):
q = coros.queue()
def do_send(q):
q.send('sent')
api.spawn(do_send, q)
self.assertEqual(q.wait(), 'sent')
def test_waiting_for_event(self):
evt = coros.event()
value = 'some stuff'
def send_to_event():
evt.send(value)
api.spawn(send_to_event)
self.assertEqual(evt.wait(), value)
def test_timer_fired(self):
def func():
call_after(0.1, self.lst.pop)
sleep(0.2)
spawn(func)
assert self.lst == [1], self.lst
sleep(0.3)
assert self.lst == [], self.lst
def test_send_last(self):
q = coros.queue()
def waiter(q):
timer = api.exc_after(0.1, api.TimeoutError)
self.assertEqual(q.wait(), 'hi2')
timer.cancel()
api.spawn(waiter, q)
api.sleep(0)
api.sleep(0)
q.send('hi2')
def test_send(self):
event1 = event()
event2 = event()
spawn(event1.send, 'hello event1')
exc_after(0, ValueError('interrupted'))
try:
result = event1.wait()
except ValueError:
X = object()
result = with_timeout(DELAY, event2.wait, timeout_value=X)
assert result is X, 'Nobody sent anything to event2 yet it received %r' % (
result, )
def test_bounded(self):
# this was originally semaphore's doctest
sem = coros.BoundedSemaphore(2, limit=3)
self.assertEqual(sem.acquire(), True)
self.assertEqual(sem.acquire(), True)
api.spawn(sem.release)
self.assertEqual(sem.acquire(), True)
self.assertEqual(-3, sem.balance)
sem.release()
sem.release()
sem.release()
api.spawn(sem.acquire)
sem.release()
self.assertEqual(3, sem.balance)
def test_waiting(self):
def do_wait(q, evt):
result = q.wait()
evt.send(result)
q = coros.queue()
e1 = coros.event()
api.spawn(do_wait, q, e1)
api.sleep(0)
self.assertEqual(1, waiting(q))
q.send('hi')
api.sleep(0)
self.assertEqual(0, waiting(q))
self.assertEqual('hi', e1.wait())
self.assertEqual(0, waiting(q))
def test_013_empty_return(self):
from eventlib import httpc
def wsgi_app(environ, start_response):
start_response("200 OK", [])
return [""]
certificate_file = os.path.join(os.path.dirname(__file__),
'test_server.crt')
private_key_file = os.path.join(os.path.dirname(__file__),
'test_server.key')
sock = api.ssl_listener(('', 4202), certificate_file, private_key_file)
api.spawn(wsgi.server, sock, wsgi_app)
res = httpc.get("https://localhost:4202/foo")
self.assertEqual(res, '')
def setUp(self):
self.logfile = StringIO()
self.victim = api.spawn(wsgi.server,
api.tcp_listener(('0.0.0.0', 31337)),
self.site_class(),
log=self.logfile,
max_size=128)
def test_blocks_on_pool(self):
waiter = coros.queue(0)
def greedy():
self.pool.get()
self.pool.get()
self.pool.get()
self.pool.get()
# No one should be waiting yet.
self.assertEquals(self.pool.waiting(), 0)
# The call to the next get will unschedule this routine.
self.pool.get()
# So this send should never be called.
waiter.send('Failed!')
killable = api.spawn(greedy)
# no one should be waiting yet.
self.assertEquals(self.pool.waiting(), 0)
## Wait for greedy
api.sleep(0)
## Greedy should be blocking on the last get
self.assertEquals(self.pool.waiting(), 1)
## Send will never be called, so balance should be 0.
self.assertFalse(waiter.ready())
api.kill(killable)
def test_killing_dormant(self):
state = []
def test():
try:
state.append('start')
sleep(DELAY)
except:
state.append('except')
# catching GreenletExit
pass
# when switching to hub, hub makes itself the parent of this greenlet,
# thus after the function's done, the control will go to the parent
# QQQ why the first sleep is not enough?
sleep(0)
state.append('finished')
g = spawn(test)
sleep(DELAY / 2)
assert state == ['start'], state
kill(g)
# will not get there, unless switching is explicitly scheduled by kill
assert state == ['start', 'except'], state
sleep(DELAY)
assert state == ['start', 'except', 'finished'], state
def test_del_closes_socket(self):
timer = api.exc_after(0.5, api.TimeoutError)
def accept_once(listener):
# delete/overwrite the original conn
# object, only keeping the file object around
# closing the file object should close everything
try:
conn, addr = listener.accept()
conn = conn.makeGreenFile()
conn.write('hello\n')
conn.close()
self.assertRaises(socket.error, conn.write, 'a')
finally:
listener.close()
server = api.tcp_listener(('0.0.0.0', 0))
killer = api.spawn(accept_once, server)
client = api.connect_tcp(('127.0.0.1', server.getsockname()[1]))
fd = client.makeGreenFile()
client.close()
assert fd.read() == b'hello\n'
assert fd.read() == ''
timer.cancel()
def test_close_with_makefile(self):
def accept_close_early(listener):
# verify that the makefile and the socket are truly independent
# by closing the socket prior to using the made file
try:
conn, addr = listener.accept()
fd = conn.makeGreenFile()
conn.close()
fd.write('hello\n')
fd.close()
self.assertRaises(socket.error, fd.write, 'a')
self.assertRaises(socket.error, conn.send, 'b')
finally:
listener.close()
def accept_close_late(listener):
# verify that the makefile and the socket are truly independent
# by closing the made file and then sending a character
try:
conn, addr = listener.accept()
fd = conn.makeGreenFile()
fd.write('hello')
fd.close()
conn.send('\n')
conn.close()
self.assertRaises(socket.error, fd.write, 'a')
self.assertRaises(socket.error, conn.send, 'b')
finally:
listener.close()
def did_it_work(server):
client = api.connect_tcp(('127.0.0.1', server.getsockname()[1]))
fd = client.makeGreenFile()
client.close()
assert fd.readline() == b'hello\n'
assert fd.read() == ''
fd.close()
server = api.tcp_listener(('0.0.0.0', 0))
killer = api.spawn(accept_close_early, server)
did_it_work(server)
api.kill(killer)
server = api.tcp_listener(('0.0.0.0', 0))
killer = api.spawn(accept_close_late, server)
did_it_work(server)
api.kill(killer)
def setUp(self):
self.logfile = StringIO()
self.site = Site()
self.killer = api.spawn(wsgi.server,
api.tcp_listener(('0.0.0.0', 12346)),
self.site,
max_size=128,
log=self.logfile)
def test_send_exc(self):
log = []
e = event()
def waiter():
try:
result = e.wait()
log.append(('received', result))
except Exception as ex:
log.append(('catched', ex))
spawn(waiter)
sleep(0) # let waiter to block on e.wait()
obj = Exception()
e.send(exc=obj)
sleep(0)
assert log == [('catched', obj)], log
def test_waiters_that_cancel(self):
q = coros.queue()
def do_receive(q, evt):
api.exc_after(0, RuntimeError())
try:
result = q.wait()
evt.send(result)
except RuntimeError:
evt.send('timed out')
evt = coros.event()
api.spawn(do_receive, q, evt)
self.assertEqual(evt.wait(), 'timed out')
q.send('hi')
self.assertEqual(q.wait(), 'hi')
def test_012_ssl_server(self):
from eventlib import httpc
def wsgi_app(environ, start_response):
start_response('200 OK', {})
return [environ['wsgi.input'].read()]
certificate_file = os.path.join(os.path.dirname(__file__),
'test_server.crt')
private_key_file = os.path.join(os.path.dirname(__file__),
'test_server.key')
sock = api.ssl_listener(('', 4201), certificate_file, private_key_file)
api.spawn(wsgi.server, sock, wsgi_app)
result = httpc.post("https://localhost:4201/foo", "abc")
self.assertEqual(result, 'abc')
def test_two_bogus_waiters(self):
def do_receive(q, evt):
api.exc_after(0, RuntimeError())
try:
result = q.wait()
evt.send(result)
except RuntimeError:
evt.send('timed out')
q = coros.queue()
e1 = coros.event()
e2 = coros.event()
api.spawn(do_receive, q, e1)
api.spawn(do_receive, q, e2)
api.sleep(0)
q.send('sent')
self.assertEqual(e1.wait(), 'timed out')
self.assertEqual(e2.wait(), 'timed out')
self.assertEqual(q.wait(), 'sent')
def test_putting_to_queue(self):
timer = api.exc_after(0.1, api.TimeoutError)
size = 2
self.pool = IntPool(min_size=0, max_size=size)
queue = coros.queue()
results = []
def just_put(pool_item, index):
self.pool.put(pool_item)
queue.send(index)
for index in xrange(size + 1):
pool_item = self.pool.get()
api.spawn(just_put, pool_item, index)
while results != range(size + 1):
x = queue.wait()
results.append(x)
timer.cancel()
def test_ssl_proxy_redirects(self):
# make sure that if the proxy returns a redirect, that httpc
# successfully follows it (this was broken at one point)
def ssl_proxy(sock):
conn, addr = sock.accept()
fd = conn.makefile()
try:
line = request = fd.readline()
self.assertEqual(request,
'GET https://localhost:1234 HTTP/1.1\r\n')
while line.strip(): # eat request headers
line = fd.readline()
# we're not going to actually proxy to localhost:1234,
# we're just going to return a response on its behalf
fd.write(
"HTTP/1.0 302 Found\r\nLocation: https://localhost:1234/2\r\n\r\n"
)
finally:
fd.close()
conn.close()
# second request, for /2 target
conn, addr = sock.accept()
fd = conn.makefile()
try:
line = request = fd.readline()
self.assertEqual(request,
'GET https://localhost:1234/2 HTTP/1.1\r\n')
while line.strip(): # eat request headers
line = fd.readline()
fd.write("HTTP/1.0 200 OK\r\n\r\n")
finally:
fd.close()
conn.close()
sock.close()
server = api.tcp_listener(('0.0.0.0', 5505))
api.spawn(ssl_proxy, server)
import os
os.environ['ALL_PROXY'] = 'localhost:5505'
httpc.get('https://localhost:1234', use_proxy=True, max_retries=1)
def __init__(self, concurrency=1):
""" Constructs an Actor, kicking off a new coroutine to process the messages.
The concurrency argument specifies how many messages the actor will try
to process concurrently. If it is 1, the actor will process messages
serially.
"""
self._mailbox = collections.deque()
self._event = event()
self._killer = api.spawn(self.run_forever)
self._pool = CoroutinePool(min_size=0, max_size=concurrency)
def __init__(self, concurrency=1):
""" Constructs an Actor, kicking off a new coroutine to process the messages.
The concurrency argument specifies how many messages the actor will try
to process concurrently. If it is 1, the actor will process messages
serially.
"""
self._mailbox = collections.deque()
self._event = event()
self._killer = api.spawn(self.run_forever)
self._pool = CoroutinePool(min_size=0, max_size=concurrency)
def execute(func, *args, **kw):
""" Executes an operation asynchronously in a new coroutine, returning
an event to retrieve the return value.
This has the same api as the CoroutinePool.execute method; the only
difference is that this one creates a new coroutine instead of drawing
from a pool.
>>> from eventlib import coros
>>> evt = coros.execute(lambda a: ('foo', a), 1)
>>> evt.wait()
('foo', 1)
"""
evt = event()
def _really_execute():
evt.send(func(*args, **kw))
api.spawn(_really_execute)
return evt
def execute(func, *args, **kw):
""" Executes an operation asynchronously in a new coroutine, returning
an event to retrieve the return value.
This has the same api as the CoroutinePool.execute method; the only
difference is that this one creates a new coroutine instead of drawing
from a pool.
>>> from eventlib import coros
>>> evt = coros.execute(lambda a: ('foo', a), 1)
>>> evt.wait()
('foo', 1)
"""
evt = event()
def _really_execute():
evt.send(func(*args, **kw))
api.spawn(_really_execute)
return evt
import sys
try:
num = int(sys.argv[1])
except:
sys.exit("Supply number of test as an argument, 0, 1, 2 or 3")
from twisted.internet import reactor
def test():
print block_on(reactor.resolver.getHostByName("www.google.com"))
print block_on(reactor.resolver.getHostByName("###"))
if num == 0:
test()
elif num == 1:
spawn(test)
from eventlib.api import sleep
print "sleeping.."
sleep(5)
print "done sleeping.."
elif num == 2:
from eventlib.twistedutil import join_reactor
spawn(test)
reactor.run()
elif num == 3:
from eventlib.twistedutil import join_reactor
print "fails because it's impossible to use block_on from the mainloop"
reactor.callLater(0, test)
def start_new_thread(function, args=(), kwargs={}):
g = spawn(function, *args, **kwargs)
return get_ident(g)
def deferToGreenThread(func, *args, **kwargs):
d = defer.Deferred()
spawn(_putResultInDeferred, d, func, args, kwargs)
return d
def callInGreenThread(func, *args, **kwargs):
return spawn(func, *args, **kwargs)