def test_send_with_recver_pref(self):
ch = greenhouse.utils.Channel()
l = [False]
m = []
n = [False]
def hotpotato():
i = ch.receive()
m.append(None)
ch.send(i)
for i in xrange(10):
greenhouse.schedule(hotpotato)
# this hot potato chain never yields to the scheduler,
# so f won't run
# terminate the hot potato. after this, f will run
@greenhouse.schedule
def g():
ch.receive()
assert len(m) == 10
assert not l[0]
n[0] = True
greenhouse.pause() # block everyone on their receive() calls
@greenhouse.schedule
def f():
l[0] = True
ch.send(None)
assert l[0]
assert n[0]
def test_exception_resiliency(self):
l = []
count = 5
def runner(item):
if item < count:
raise AttributeError("blah")
l.append(item**2)
pool = self.POOL(runner, count)
pool.start()
for i in xrange(count):
pool.put(i)
greenhouse.pause()
self.assertEqual(l, [])
for i in xrange(count, count * 3):
pool.put(i)
greenhouse.pause()
self.assertEqual(sorted(l), [x**2 for x in xrange(count, count * 3)])
pool.close()
def handle_map(self, job_id, dumped_datasource, dumped_mapper, reducer_id):
ds_func = load_func(dumped_datasource)
items = ds_func(self._mapper_id)
map_func = load_func(dumped_mapper)
results = []
for i, item in enumerate(items):
results.append(map_func(item))
if i and not i % self.page_size:
self._hub.publish(
"%s-reducer" % (self._service,),
reducer_id,
"reduce",
(job_id, results, False),
{})
results = []
greenhouse.pause()
self._hub.publish(
"%s-reducer" % (self._service,),
reducer_id,
"reduce",
(job_id, results, True),
{})
def test_exception_resiliency(self):
l = []
count = 5
def runner(item):
if item < count:
raise AttributeError("blah")
l.append(item ** 2)
pool = self.POOL(runner, count)
pool.start()
for i in xrange(count):
pool.put(i)
greenhouse.pause()
self.assertEqual(l, [])
for i in xrange(count, count * 3):
pool.put(i)
greenhouse.pause()
self.assertEqual(sorted(l), [x ** 2 for x in xrange(count, count * 3)])
pool.close()
def test_local_exception_handlers(self):
class CustomError1(Exception):
pass
class CustomError2(Exception):
pass
l = []
def handler(klass, exc, tb):
l.append(klass)
@greenhouse.schedule
@greenhouse.greenlet
def g1():
raise CustomError1()
@greenhouse.schedule
@greenhouse.greenlet
def g2():
raise CustomError2()
greenhouse.local_exception_handler(handler, g2)
greenhouse.pause()
self.assertEqual(len(l), 1)
self.assertEqual(l[0], CustomError2)
def test_kills_all_coros(self):
class Pool(self.POOL):
def __init__(self, *args, **kwargs):
super(Pool, self).__init__(*args, **kwargs)
self.finished = []
def _runner(self):
super(Pool, self)._runner()
self.finished.append(None)
def f(x):
return x ** 2
pool = Pool(f)
pool.start()
for x in xrange(30):
pool.put(x)
self.empty_out(pool, 30)
pool.close()
greenhouse.pause()
self.assertEqual(len(pool.finished), pool.size)
def test_async_rpc_success(self):
handler_results = []
sender_results = []
def handler(x):
handler_results.append(x)
return x ** 2
self.peer.accept_rpc("service", 0, 0, "method", handler)
for i in xrange(4):
greenhouse.pause()
rpcs = []
rpcs.append(self.sender.send_rpc("service", 0, "method", (1,), {}))
rpcs.append(self.sender.send_rpc("service", 0, "method", (2,), {}))
rpcs.append(self.sender.send_rpc("service", 0, "method", (3,), {}))
rpcs.append(self.sender.send_rpc("service", 0, "method", (4,), {}))
while rpcs:
rpc = junction.wait_any(rpcs, TIMEOUT)
rpcs.remove(rpc)
sender_results.append(rpc.value)
self.assertEqual(rpcs, [])
self.assertEqual(handler_results, [1, 2, 3, 4])
self.assertEqual(sender_results, [1, 4, 9, 16])
def test_schedule_at(self):
at = time.time() + (TESTING_TIMEOUT * 2)
l = []
def f1():
l.append(1)
greenhouse.schedule_at(at, f1)
@greenhouse.schedule_at(at)
def f2():
l.append(2)
@greenhouse.schedule_at(at, args=(3,))
def f3(x):
l.append(x)
@greenhouse.schedule_at(at, kwargs={'x': 4})
def f4(x=None):
l.append(x)
@greenhouse.compat.greenlet
def f5():
l.append(5)
greenhouse.schedule_at(at, f5)
greenhouse.pause()
assert not l
greenhouse.pause_for(TESTING_TIMEOUT * 2)
assert time.time() >= at
greenhouse.pause()
l.sort()
assert l == [1, 2, 3, 4, 5], l
def test_local_exception_handlers(self):
class CustomError1(Exception):
pass
class CustomError2(Exception):
pass
l = []
def handler(klass, exc, tb):
l.append(klass)
@greenhouse.schedule
@greenhouse.greenlet
def g1():
raise CustomError1()
@greenhouse.schedule
@greenhouse.greenlet
def g2():
raise CustomError2()
greenhouse.local_exception_handler(handler, g2)
greenhouse.pause()
self.assertEqual(len(l), 1)
self.assertEqual(l[0], CustomError2)
def serve_all(self):
self.start_timer()
while not self.closing:
request = self.get_request()
self.cancel_timer()
if request is None:
# indicates connection terminated by client
break
handler = self.request_handler(
self.client_address, self.server_address, self)
# the return value from handler.handle may be a generator or
# other lazy iterator to allow for large responses that send
# in chunks and don't block the entire server the whole time
response = handler.handle(request)
first = True
for chunk in response:
if not first:
greenhouse.pause()
self.socket.sendall(chunk)
first = False
self.start_timer()
greenhouse.pause()
self.cleanup()
def test_basic_communication(self):
collector = []
channel = greenhouse.utils.Channel()
recver = self.recver(channel, collector)
greenhouse.schedule(recver)
greenhouse.schedule(recver)
greenhouse.schedule(recver)
greenhouse.schedule(recver)
greenhouse.schedule(recver)
greenhouse.pause()
greenhouse.schedule(channel.send, (3,))
greenhouse.schedule(channel.send, (4,))
greenhouse.schedule(channel.send, (5,))
greenhouse.pause()
assert collector == [3,4,5], collector
channel.send(6)
channel.send(7)
assert collector == [3,4,5,6,7], collector
def test_local_outgoing_trace_hook(self):
@greenhouse.schedule
@greenhouse.greenlet
def g():
for i in xrange(4):
greenhouse.pause()
raise AttributeError("BWAHAHA")
l = [0]
@greenhouse.local_outgoing_trace_hook(coro=g)
def hook(direction, coro):
l[0] += 1
m = [0]
@greenhouse.local_exception_handler(coro=g)
def handler(klass, exc, tb):\
m[0] += 1
for i in xrange(4):
greenhouse.pause()
self.assertEqual(l[0], i + 1)
greenhouse.pause()
# g exits via exception, outgoing hook still runs
self.assertEqual(m[0], 1)
self.assertEqual(l[0], 5)
def test_join(self):
ev = greenhouse.util.Event()
class T(greenhouse.util.Thread):
def run(self):
ev.wait()
threads = [T() for i in xrange(5)]
[t.start() for t in threads]
l = []
@greenhouse.schedule
def joiner():
threads[2].join()
l.append(None)
greenhouse.pause_for(TESTING_TIMEOUT)
self.assertEqual(len(l), 0)
ev.set()
greenhouse.pause()
greenhouse.pause()
self.assertEqual(len(l), 1)
def test_getters_raise_on_close(self):
l = []
raised = [False]
def runner(item):
return item ** 2
pool = self.POOL(runner, 5)
pool.start()
@greenhouse.schedule
def getter():
try:
while 1:
l.append(pool.get())
except StandardError:
raised[0] = True
for i in xrange(10):
pool.put(i)
greenhouse.pause()
self.assertEqual(l, [x ** 2 for x in xrange(10)])
pool.close()
greenhouse.pause()
assert raised[0]
def test_args_and_kwargs(self):
s = set()
class WithArgs(greenhouse.util.Thread):
def run(self, num):
s.add(num)
a1 = WithArgs(args=(1,))
a2 = WithArgs(args=(2,))
a3 = WithArgs(args=(3,))
a1.start()
a2.start()
a3.start()
class WithKwargs(greenhouse.util.Thread):
def run(self, **kwargs):
s.update(kwargs.keys())
k1 = WithKwargs(kwargs={'foo': 1})
k2 = WithKwargs(kwargs={'bar': 1})
k1.start()
k2.start()
self.assertEqual(s, set())
greenhouse.pause()
self.assertEqual(s, set([1, 2, 3, 'foo', 'bar']))
def test_args_and_kwargs(self):
s = set()
class WithArgs(greenhouse.util.Thread):
def run(self, num):
s.add(num)
a1 = WithArgs(args=(1, ))
a2 = WithArgs(args=(2, ))
a3 = WithArgs(args=(3, ))
a1.start()
a2.start()
a3.start()
class WithKwargs(greenhouse.util.Thread):
def run(self, **kwargs):
s.update(kwargs.keys())
k1 = WithKwargs(kwargs={'foo': 1})
k2 = WithKwargs(kwargs={'bar': 1})
k1.start()
k2.start()
self.assertEqual(s, set())
greenhouse.pause()
self.assertEqual(s, set([1, 2, 3, 'foo', 'bar']))
def test_join(self):
ev = greenhouse.util.Event()
class T(greenhouse.util.Thread):
def run(self):
ev.wait()
threads = [T() for i in xrange(5)]
[t.start() for t in threads]
l = []
@greenhouse.schedule
def joiner():
threads[2].join()
l.append(None)
greenhouse.pause_for(TESTING_TIMEOUT)
self.assertEqual(len(l), 0)
ev.set()
greenhouse.pause()
greenhouse.pause()
self.assertEqual(len(l), 1)
def test_getters_raise_on_close(self):
l = []
raised = [False]
def runner(item):
return item**2
pool = self.POOL(runner, 5)
pool.start()
@greenhouse.schedule
def getter():
try:
while 1:
l.append(pool.get())
except StandardError:
raised[0] = True
for i in xrange(10):
pool.put(i)
greenhouse.pause()
self.assertEqual(l, [x**2 for x in xrange(10)])
pool.close()
greenhouse.pause()
assert raised[0]
def test_kills_all_coros(self):
class Pool(self.POOL):
def __init__(self, *args, **kwargs):
super(Pool, self).__init__(*args, **kwargs)
self.finished = []
def _runner(self):
super(Pool, self)._runner()
self.finished.append(None)
def f(x):
return x**2
pool = Pool(f)
pool.start()
for x in xrange(30):
pool.put(x)
self.empty_out(pool, 30)
pool.close()
greenhouse.pause()
self.assertEqual(len(pool.finished), pool.size)
def test_schedule_in(self):
l = []
def f1():
l.append(1)
greenhouse.schedule_in(TESTING_TIMEOUT, f1)
@greenhouse.schedule_in(TESTING_TIMEOUT)
def f2():
l.append(2)
@greenhouse.schedule_in(TESTING_TIMEOUT, args=(3,))
def f3(x):
l.append(x)
@greenhouse.schedule_in(TESTING_TIMEOUT, kwargs={'x': 4})
def f4(x=None):
l.append(x)
@greenhouse.compat.greenlet
def f5():
l.append(5)
greenhouse.schedule_in(TESTING_TIMEOUT, f5)
greenhouse.pause()
assert not l
time.sleep(TESTING_TIMEOUT)
greenhouse.pause()
l.sort()
assert l == [1, 2, 3, 4, 5], l
def test_starting_back_up(self):
def f(x):
return x**2
pool = self.POOL(f)
pool.start()
for x in xrange(30):
pool.put(x)
self.empty_out(pool, 30)
pool.close()
greenhouse.pause()
pool.start()
for x in xrange(30):
pool.put(x)
l = self.empty_out(pool, 30)
l.sort()
assert l == [x**2 for x in xrange(30)]
def test_schedule_at(self):
at = time.time() + (TESTING_TIMEOUT * 2)
l = []
def f1():
l.append(1)
greenhouse.schedule_at(at, f1)
@greenhouse.schedule_at(at)
def f2():
l.append(2)
@greenhouse.schedule_at(at, args=(3,))
def f3(x):
l.append(x)
@greenhouse.schedule_at(at, kwargs={'x': 4})
def f4(x=None):
l.append(x)
@greenhouse.compat.greenlet
def f5():
l.append(5)
greenhouse.schedule_at(at, f5)
greenhouse.pause()
assert not l
greenhouse.pause_for(TESTING_TIMEOUT * 2)
assert time.time() >= at
greenhouse.pause()
l.sort()
assert l == [1, 2, 3, 4, 5], l
def test_local_outgoing_hook(self):
@greenhouse.schedule
@greenhouse.greenlet
def g():
for i in xrange(4):
greenhouse.pause()
raise AttributeError("BWAHAHA")
l = [0]
@greenhouse.local_outgoing_hook(coro=g)
def hook(direction, coro):
l[0] += 1
m = [0]
@greenhouse.local_exception_handler(coro=g)
def handler(klass, exc, tb):\
m[0] += 1
for i in xrange(4):
greenhouse.pause()
self.assertEqual(l[0], i + 1)
greenhouse.pause()
# g exits via exception, outgoing hook still runs
self.assertEqual(m[0], 1)
self.assertEqual(l[0], 5)
def test_local_incoming_hook(self):
lock1 = greenhouse.Lock()
lock2 = greenhouse.Lock()
@greenhouse.schedule
@greenhouse.greenlet
def g1():
with lock1:
for i in xrange(3):
greenhouse.pause()
l1 = [0]
l2 = [0]
@greenhouse.schedule
@greenhouse.greenlet
def g2():
with lock2:
for i in xrange(5):
greenhouse.pause()
@greenhouse.local_incoming_hook(coro=g1)
def handler1(direction, coroutine):
l1[0] += 1
@greenhouse.local_incoming_hook(coro=g2)
def handler2(direction, coroutine):
l2[0] += 1
greenhouse.pause()
lock1.acquire()
lock2.acquire()
self.assertEqual(l1, [4])
self.assertEqual(l2, [6])
def test_schedule_in(self):
l = []
def f1():
l.append(1)
greenhouse.schedule_in(TESTING_TIMEOUT, f1)
@greenhouse.schedule_in(TESTING_TIMEOUT)
def f2():
l.append(2)
@greenhouse.schedule_in(TESTING_TIMEOUT, args=(3,))
def f3(x):
l.append(x)
@greenhouse.schedule_in(TESTING_TIMEOUT, kwargs={'x': 4})
def f4(x=None):
l.append(x)
@greenhouse.compat.greenlet
def f5():
l.append(5)
greenhouse.schedule_in(TESTING_TIMEOUT, f5)
greenhouse.pause()
assert not l
time.sleep(TESTING_TIMEOUT)
greenhouse.pause()
l.sort()
assert l == [1, 2, 3, 4, 5], l
def test_local_incoming_trace_hook(self):
lock1 = greenhouse.Lock()
lock2 = greenhouse.Lock()
@greenhouse.schedule
@greenhouse.greenlet
def g1():
with lock1:
for i in xrange(3):
greenhouse.pause()
l1 = [0]
l2 = [0]
@greenhouse.schedule
@greenhouse.greenlet
def g2():
with lock2:
for i in xrange(5):
greenhouse.pause()
@greenhouse.local_incoming_trace_hook(coro=g1)
def handler1(direction, coroutine):
l1[0] += 1
@greenhouse.local_incoming_trace_hook(coro=g2)
def handler2(direction, coroutine):
l2[0] += 1
greenhouse.pause()
lock1.acquire()
lock2.acquire()
self.assertEqual(l1, [4])
self.assertEqual(l2, [6])
def test_starting_back_up(self):
def f(x):
return x ** 2
pool = self.POOL(f)
pool.start()
for x in xrange(30):
pool.put(x)
self.empty_out(pool, 30)
pool.close()
greenhouse.pause()
pool.start()
for x in xrange(30):
pool.put(x)
l = self.empty_out(pool, 30)
l.sort()
assert l == [x ** 2 for x in xrange(30)]
def test_returns_false_nonblocking(self):
lock = self.LOCK()
@greenhouse.schedule
def f():
assert not lock.acquire(blocking=False)
lock.acquire()
greenhouse.pause()
def test_schedule_exception_in_rejects_dead_glets(self):
@greenhouse.schedule
@greenhouse.greenlet
def g():
pass
greenhouse.pause()
self.assertRaises(
ValueError, greenhouse.schedule_exception_in, 1, Exception(), g)
def test_schedule_exception_in_rejects_dead_glets(self):
@greenhouse.schedule
@greenhouse.greenlet
def g():
pass
greenhouse.pause()
self.assertRaises(
ValueError, greenhouse.schedule_exception_in, 1, Exception(), g)
def test_returns_false_nonblocking(self):
lock = self.LOCK()
@greenhouse.schedule
def f():
assert not lock.acquire(blocking=False)
lock.acquire()
greenhouse.pause()
def test_end_permits_dead_glets(self):
@greenhouse.schedule
@greenhouse.greenlet
def g():
pass
greenhouse.pause()
# no assert, just make sure we don't raise
greenhouse.end(g)
def test_end_permits_dead_glets(self):
@greenhouse.schedule
@greenhouse.greenlet
def g():
pass
greenhouse.pause()
# no assert, just make sure we don't raise
greenhouse.end(g)
def test_rpc_ruled_out_by_method(self):
results = []
self.peer.accept_rpc("service", 0, 0, "method1", results.append)
for i in xrange(4):
greenhouse.pause()
self.assertRaises(junction.errors.UnsupportedRemoteMethod,
self.sender.rpc, "service", 0, "method2", (1,), {}, TIMEOUT)
def test_socket_timeout_in_grlet(self):
with self.socketpair() as (client, handler):
client.settimeout(TESTING_TIMEOUT)
assert client.gettimeout() == TESTING_TIMEOUT
@greenhouse.schedule
def f():
self.assertRaises(socket.timeout, client.recv, 10)
client.recv(10)
greenhouse.pause()
def test_pause(self):
l = [False]
@greenhouse.schedule
def f():
l[0] = True
assert not l[0]
greenhouse.pause()
assert l[0]
def test_pause(self):
l = [False]
@greenhouse.schedule
def f():
l[0] = True
assert not l[0]
greenhouse.pause()
assert l[0]
def test_socket_timeout_in_grlet(self):
with self.socketpair() as (client, handler):
client.settimeout(TESTING_TIMEOUT)
assert client.gettimeout() == TESTING_TIMEOUT
@greenhouse.schedule
def f():
self.assertRaises(socket.timeout, client.recv, 10)
client.recv(10)
greenhouse.pause()
def test_timeout_callback(self):
ev = greenhouse.Event()
l = [False]
@ev._add_timeout_callback
def c():
l[0] = True
ev.wait(TESTING_TIMEOUT)
greenhouse.pause()
assert l[0]
def test_rpc_ruled_out_by_routing_id(self):
results = []
self.peer.accept_rpc("service", 1, 0, "method", results.append)
for i in xrange(4):
greenhouse.pause()
with self.assertRaises(junction.errors.Unroutable):
self.sender.rpc("service", 1, "method", (1,), timeout=TIMEOUT)
self.assertEqual(results, [])
def test_nonblocking_when_set(self):
ev = greenhouse.Event()
l = [False]
ev.set()
@greenhouse.schedule
def f():
ev.wait()
l[0] = True
greenhouse.pause()
assert l[0]
def test_pauses(self):
l = [False]
def f():
l[0] = True
timer = greenhouse.Timer(TESTING_TIMEOUT, f)
assert not l[0]
time.sleep(TESTING_TIMEOUT)
greenhouse.pause()
assert l[0]
def test_kwargs(self):
d = {}
class WithKwargs(greenhouse.util.Thread):
def run(self, **kwargs):
d.update(kwargs)
threads = [WithKwargs(kwargs={chr(x): x}) for x in xrange(97, 123)]
[t.start() for t in threads]
greenhouse.pause()
self.assertEqual(d, dict((chr(x), x) for x in xrange(97, 123)))
def test_nonblocking_when_set(self):
ev = greenhouse.Event()
l = [False]
ev.set()
@greenhouse.schedule
def f():
ev.wait()
l[0] = True
greenhouse.pause()
assert l[0]
def test_attribute_error(self):
#1
loc = greenhouse.Local()
@greenhouse.schedule
def f():
# 2
loc.foo = "f"
greenhouse.pause() # to 3
greenhouse.pause() # to 2
# 3
self.assertRaises(AttributeError, lambda: loc.foo)
def test_args(self):
s = set()
class WithArgs(greenhouse.util.Thread):
def run(self, num):
s.add(num)
threads = [WithArgs(args=(x, )) for x in xrange(5)]
[t.start() for t in threads]
greenhouse.pause()
self.assertEqual(s, set(xrange(5)))
def test_attribute_error(self):
#1
loc = greenhouse.Local()
@greenhouse.schedule
def f():
# 2
loc.foo = "f"
greenhouse.pause() # to 3
greenhouse.pause() # to 2
# 3
self.assertRaises(AttributeError, lambda: loc.foo)
def test_args(self):
s = set()
class WithArgs(greenhouse.util.Thread):
def run(self, num):
s.add(num)
threads = [WithArgs(args=(x,)) for x in xrange(5)]
[t.start() for t in threads]
greenhouse.pause()
self.assertEqual(s, set(xrange(5)))
def test_order(self):
l = []
@greenhouse.schedule
def f():
l.append(1)
@greenhouse.schedule
def f():
l.append(2)
greenhouse.pause()
self.assertEqual(l, [1, 2])
def test_blocks(self):
c = util.Counter()
c.increment()
l = [0]
@greenhouse.schedule
def f():
c.wait()
l[0] += 1
greenhouse.pause()
self.assertEqual(l[0], 0)
def test_sockets_btwn_grlets(self):
with self.socketpair() as (client, handler):
grlet_results = []
@greenhouse.schedule
def f():
client.send("hello from a greenlet")
grlet_results.append(client.recv(19))
assert handler.recv(21) == "hello from a greenlet"
handler.send("hello to a greenlet")
greenhouse.pause()
assert grlet_results[0] == "hello to a greenlet"
def serve(self):
if not self.is_setup:
self.setup()
self.ready.set()
try:
while not self.shutting_down:
self.handle_packet(*self.socket.recvfrom(self.max_packet_size))
if not self.shutting_down:
greenhouse.pause()
except KeyboardInterrupt:
pass
finally:
self.cleanup()
def test_release_race(self):
lock = self.LOCK()
l = []
@greenhouse.schedule
def f():
lock.acquire()
greenhouse.pause()
lock.release()
@greenhouse.schedule
def g1():
lock.acquire()
l.append(1)
@greenhouse.schedule
def g2():
lock.acquire()
l.append(2)
# let f grab the lock and the gs get blocked
greenhouse.pause()
# this time f releases, and it should only wake up one of the gs
greenhouse.pause()
greenhouse.pause()
greenhouse.pause()
self.assertEquals(len(l), 1)
def test_order(self):
l = []
@greenhouse.schedule
def f():
l.append(1)
@greenhouse.schedule
def f():
l.append(2)
greenhouse.pause()
self.assertEqual(l, [1, 2])
def test_timeouts_in_grlets(self):
l = [False]
ev = greenhouse.Event()
@greenhouse.schedule
def f():
ev.wait(TESTING_TIMEOUT)
l[0] = True
greenhouse.pause()
assert not l[0]
greenhouse.pause_for(TESTING_TIMEOUT * 2)
assert l[0]
def test_owned_by_someone_else(self):
cond = greenhouse.Condition(self.LOCK())
@greenhouse.schedule
def f():
cond.acquire()
greenhouse.pause()
greenhouse.pause()
self.assertRaises(RuntimeError, cond.notify)
self.assertRaises(RuntimeError, cond.wait)
self.assertRaises(RuntimeError, cond.notify_all)
greenhouse.pause()
def test_end(self):
l = [False]
@greenhouse.schedule
@greenhouse.greenlet
def glet():
greenhouse.pause()
l[0] = True
greenhouse.pause()
greenhouse.end(glet)
greenhouse.pause()
assert not l[0]