def test_unrelated_rpcs_are_unaffected(self):
global PORT
hub = junction.Hub(("127.0.0.1", PORT), [])
PORT += 2
@hub.accept_rpc('service', 0, 0, 'method')
def handle():
greenhouse.pause_for(TIMEOUT)
return 1
hub.start()
peer = junction.Hub(("127.0.0.1", PORT), [hub.addr])
PORT += 2
peer.start()
peer.wait_connected()
client = junction.Client(hub.addr)
client.connect()
client.wait_connected()
client = [client]
greenhouse.schedule(self.kill_client, (client,))
# hub does a self-rpc during which the client connection goes away
result = peer.rpc('service', 0, 'method')
self.assertEqual(result, 1)
def test_schedule(self):
l = []
def f1():
l.append(1)
greenhouse.schedule(f1)
@greenhouse.schedule
def f2():
l.append(2)
@greenhouse.schedule(args=(3,))
def f3(x):
l.append(x)
@greenhouse.schedule(kwargs={'x': 4})
def f4(x=None):
l.append(x)
@greenhouse.compat.greenlet
def f5():
l.append(5)
greenhouse.schedule(f5)
greenhouse.pause_for(TESTING_TIMEOUT)
l.sort()
assert l == [1, 2, 3, 4, 5], l
def finish(self, value):
'''Give the future it's value and trigger any success callbacks
:param value: the new value for the future
:raises:
:class:`AlreadyComplete <zugzug.errors.AlreadyComplete>` if
already complete
'''
if self._done.is_set():
raise errors.AlreadyComplete()
self._value = value
for cb in self._cbacks:
greenhouse.schedule(cb, args=(value, ))
self._cbacks = None
for wait in list(self._waits):
wait.finish(self)
self._waits = None
for child in self._children:
child = child()
if child is None:
continue
child._incoming(self)
self._children = None
self._done.set()
def finish(self, value):
'''Give the future it's value and trigger any success callbacks
:param value: the new value for the future
:raises:
:class:`AlreadyComplete <zugzug.errors.AlreadyComplete>` if
already complete
'''
if self._done.is_set():
raise errors.AlreadyComplete()
self._value = value
for cb in self._cbacks:
greenhouse.schedule(cb, args=(value,))
self._cbacks = None
for wait in list(self._waits):
wait.finish(self)
self._waits = None
for child in self._children:
child = child()
if child is None:
continue
child._incoming(self)
self._children = None
self._done.set()
def _incoming(self, parent):
index = self._parent_indexes.pop(parent)
self._parents[index] = None
self._parent_results[index] = parent._value
if all(p is None for p in self._parents):
greenhouse.schedule(self._run_func)
def test_send_with_recver_pref(self):
ch = greenhouse.utils.Channel()
l = [False]
m = []
n = [False]
def hotpotato():
i = ch.receive()
m.append(None)
ch.send(i)
for i in xrange(10):
greenhouse.schedule(hotpotato)
# this hot potato chain never yields to the scheduler,
# so f won't run
# terminate the hot potato. after this, f will run
@greenhouse.schedule
def g():
ch.receive()
assert len(m) == 10
assert not l[0]
n[0] = True
greenhouse.pause() # block everyone on their receive() calls
@greenhouse.schedule
def f():
l[0] = True
ch.send(None)
assert l[0]
assert n[0]
def test_schedule(self):
l = []
def f1():
l.append(1)
greenhouse.schedule(f1)
@greenhouse.schedule
def f2():
l.append(2)
@greenhouse.schedule(args=(3,))
def f3(x):
l.append(x)
@greenhouse.schedule(kwargs={'x': 4})
def f4(x=None):
l.append(x)
@greenhouse.compat.greenlet
def f5():
l.append(5)
greenhouse.schedule(f5)
greenhouse.pause_for(TESTING_TIMEOUT)
l.sort()
assert l == [1, 2, 3, 4, 5], l
def _incoming(self, parent):
index = self._parent_indexes.pop(parent)
self._parents[index] = None
self._parent_results[index] = parent._value
if all(p is None for p in self._parents):
greenhouse.schedule(self._run_func)
def get_urls(urls):
count = len(urls)
results = {}
alldone = greenhouse.Event()
# each url gets its own greenlet to fetch it
for url in urls:
greenhouse.schedule(_get_one, args=(url, results, count, alldone))
alldone.wait()
return results
def main():
hub = junction.Hub(("localhost", PORT), [("localhost", SERVICE_PORT)])
hub.start()
greenhouse.schedule(greenhouse.run_backdoor,
args=(("localhost", PORT + 1), {'hub': hub}))
try:
greenhouse.Event().wait()
except KeyboardInterrupt:
pass
def main():
hub.start()
hub.wait_connected()
greenhouse.schedule(greenhouse.run_backdoor,
args=((BDHOST, BDPORT), {'hub': hub}))
try:
greenhouse.Event().wait()
except KeyboardInterrupt:
pass
def get_urls(urls):
count = len(urls)
results = {}
alldone = greenhouse.Event()
# each url gets its own greenlet to fetch it
for url in urls:
greenhouse.schedule(_get_one, args=(url, results, count, alldone))
alldone.wait()
return results
def main():
hub = junction.Hub((HOST, PORT), [(RELAY_HOST, RELAY_PORT)])
hub.start()
hub.accept_rpc(SERVICE, 0, 0, "echo", handler)
greenhouse.schedule(greenhouse.run_backdoor,
args=((BACKHOST, BACKPORT), {'hub': hub}))
try:
greenhouse.Event().wait()
except KeyboardInterrupt:
pass
def main():
hub = junction.Hub(("localhost", PORT), [("localhost", MIDDLEMAN_PORT)])
hub.start()
hub.accept_rpc(WAIT_SERVICE, 0, 0, "wait", wait)
greenhouse.schedule(greenhouse.run_backdoor,
args=(("localhost", PORT + 1), {'hub': hub}))
try:
greenhouse.Event().wait()
except KeyboardInterrupt:
pass
def greenhouse_serve():
greenhouse.set_ignore_interrupts(1)
servsock = greenhouse.Socket()
servsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
servsock.bind(("127.0.0.1", 8000))
servsock.listen(socket.SOMAXCONN)
try:
while 1:
client, addr = servsock.accept()
greenhouse.schedule(greenhouse_handler, (client,))
except GoAway:
pass
def greenhouse_serve():
greenhouse.set_ignore_interrupts(1)
servsock = greenhouse.Socket()
servsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
servsock.bind(("127.0.0.1", 8000))
servsock.listen(socket.SOMAXCONN)
try:
while 1:
client, addr = servsock.accept()
greenhouse.schedule(greenhouse_handler, (client, ))
except GoAway:
pass
def main():
hub = junction.Hub(("localhost", PORT), [("localhost", SERVICE_PORT)])
hub.start()
greenhouse.schedule(greenhouse.run_backdoor,
args=(("localhost", PORT + 1), {
'hub': hub
}))
try:
greenhouse.Event().wait()
except KeyboardInterrupt:
pass
def main():
print "localhost echoing server starting on port %d." % PORT
print "shut it down with <Ctrl>-C"
try:
serversock = greenhouse.Socket()
serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversock.bind(("", PORT))
serversock.listen(5)
while 1:
clientsock, address = serversock.accept()
greenhouse.schedule(connection_handler, args=[clientsock, address])
except KeyboardInterrupt:
print "KeyboardInterrupt caught, closing listener socket"
serversock.close()
def http_server(self, handler, bind_addr="127.0.0.1", port=9999):
class Connection(http.HTTPConnection):
request_handler = handler
server = http.HTTPServer((bind_addr, port))
server.connection_handler = Connection
server.worker_count = 1
greenhouse.schedule(server.serve)
greenhouse.pause()
yield server
server.socket.shutdown(socket.SHUT_RDWR)
greenhouse.pause()
def main(environ, argv):
finished = greenhouse.Event()
backdoor = greenhouse.greenlet(run_backdoor, args=(finished,))
greenhouse.schedule(backdoor)
# nothing in run_backdoor blocks until it succeeds in
# getting a port, so this can just be a simple pause
greenhouse.pause()
telnet = subprocess.Popen(['telnet', '127.0.0.1', str(BACKDOOR_PORT)])
rc = telnet.wait()
greenhouse.end(backdoor)
finished.wait()
return rc
def main():
hub = junction.Hub(("localhost", PORT), [("localhost", MIDDLEMAN_PORT)])
hub.start()
hub.accept_rpc(WAIT_SERVICE, 0, 0, "wait", wait)
greenhouse.schedule(greenhouse.run_backdoor,
args=(("localhost", PORT + 1), {
'hub': hub
}))
try:
greenhouse.Event().wait()
except KeyboardInterrupt:
pass
def main():
hub = junction.Hub((HOST, PORT), [(RELAY_HOST, RELAY_PORT)])
hub.start()
hub.accept_rpc(SERVICE, 0, 0, "echo", handler)
greenhouse.schedule(greenhouse.run_backdoor,
args=((BACKHOST, BACKPORT), {
'hub': hub
}))
try:
greenhouse.Event().wait()
except KeyboardInterrupt:
pass
def wsgi_server(self, app, bind_addr="127.0.0.1", port=9999):
class RequestHandler(wsgi.WSGIHTTPRequestHandler):
wsgiapp = app
class Connection(http.HTTPConnection):
request_handler = RequestHandler
server = http.HTTPServer((bind_addr, port))
server.connection_handler = Connection
server.worker_count = 1
greenhouse.schedule(server.serve)
greenhouse.pause()
yield server
server.socket.shutdown(socket.SHUT_RDWR)
greenhouse.pause()
def test_unrelated_client_chunked_publishes_are_unrelated(self):
global PORT
hub = junction.Hub(("127.0.0.1", PORT), [])
PORT += 2
d = {}
@hub.accept_publish('service', 0, 0, 'method')
def handle(x, source):
for item in x:
d.setdefault(source, 0)
d[source] += 1
hub.start()
c1 = junction.Client(("127.0.0.1", PORT - 2))
c1.connect()
c1.wait_connected()
c2 = junction.Client(("127.0.0.1", PORT - 2))
c2.connect()
c2.wait_connected()
def gen():
greenhouse.pause_for(TIMEOUT)
yield None
greenhouse.pause_for(TIMEOUT)
yield None
greenhouse.pause_for(TIMEOUT)
yield None
greenhouse.schedule(c1.publish, args=('service', 0, 'method'),
kwargs={'args': (gen(),), 'kwargs': {'source': 'a'}})
greenhouse.schedule(c2.publish, args=('service', 0, 'method'),
kwargs={'args': (gen(),), 'kwargs': {'source': 'b'}})
greenhouse.pause_for(TIMEOUT)
c2 = [c2]
self.kill_client(c2)
greenhouse.pause_for(TIMEOUT)
greenhouse.pause_for(TIMEOUT)
greenhouse.pause_for(TIMEOUT)
self.assertEquals(d, {'a': 3, 'b': 1})
def start(self):
read_req, write_req = os.pipe()
read_resp, write_resp = os.pipe()
pid = os.fork()
if pid: # in master proc
self.pid = pid
_ungreenify(read_resp)
_ungreenify(write_req)
self.readpipe = read_resp
self.writepipe = write_req
self._childpipes = (read_req, write_resp)
assert green_os.read(read_resp, 1) == '\x00'
#TODO: multiplex responses in a single collector in the master
self._collector = greenhouse.greenlet(self._collect)
greenhouse.schedule(self._collector)
else: # in worker proc
_ungreenify(read_req)
_ungreenify(write_resp)
self.readpipe = read_req
self.writepipe = write_resp
greenhouse.unpatch()
while not os.write(write_resp, '\x00'):
pass
while 1:
cmd, args, kwargs = recv_req(read_req)
if cmd == CMD_JOB:
try:
result = self.func(*args, **kwargs)
except Exception:
trace = traceback.format_exception(*sys.exc_info())
send_resp(RC_BAD, ''.join(trace).strip())
else:
send_resp(write_resp, RC_GOOD, result)
elif cmd == CMD_CLOSE:
sys.exit(0)
def serve(self):
"""run the server at the provided address forever.
this method will remove the calling greenlet (generally the main
greenlet) from the scheduler, so don't expect anything else to run in
the calling greenlet until the server has been shut down.
"""
if not self.is_setup:
self.setup()
try:
while not self.shutting_down:
try:
client_sock, client_address = self.socket.accept()
handler = self.connection_handler(
client_sock,
client_address,
self.address,
self.killable)
greenhouse.schedule(handler.serve_all)
except socket.error, error:
if err.args[0] == errno.EMFILE:
# max open connections for the process
if not self.killable:
# if all connections are active, just wait a
# while before accepting a new connection again
greenhouse.pause_for(0.01)
continue
# close all the connections that are
# only open for keep-alive anyway
for fd in self.killable.keys():
handler = self.killable.pop(fd)
handler.socket.close()
handler.closed = True
elif err.args[0] == errno.ENFILE:
# max open connections for the machine
greenhouse.pause_for(0.01)
else:
raise
except KeyboardInterrupt:
pass
finally:
self.socket.close()
def start():
print "localhost nc chat server starting on port %d." % PORT
print "shut it down with <Ctrl>-C"
try:
serversock = greenhouse.Socket()
serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversock.bind(("", PORT))
serversock.listen(5)
while 1:
clientsock, address = serversock.accept()
greenhouse.schedule(connection_handler, args=(clientsock,))
except KeyboardInterrupt:
print "KeyboardInterrupt caught, closing connections"
serversock.close()
for sock in CONNECTED.values():
sock.close()
def start():
print "localhost nc chat server starting on port %d." % PORT
print "shut it down with <Ctrl>-C"
try:
serversock = greenhouse.Socket()
serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversock.bind(("", PORT))
serversock.listen(5)
while 1:
clientsock, address = serversock.accept()
greenhouse.schedule(connection_handler, args=(clientsock, ))
except KeyboardInterrupt:
print "KeyboardInterrupt caught, closing connections"
serversock.close()
for sock in CONNECTED.values():
sock.close()
def serve(self):
"""run the server at the provided address forever.
this method will remove the calling greenlet (generally the main
greenlet) from the scheduler, so don't expect anything else to run in
the calling greenlet until the server has been shut down.
"""
if self.daemonize and os.environ.get('DAEMON', None) != 'yes':
os.environ['DAEMON'] = 'yes'
util.background()
if not self.is_setup:
self.setup()
self.ready.set()
try:
while not self.shutting_down:
try:
client_sock, client_address = self.socket.accept()
handler = self.connection_handler(
client_sock,
client_address,
self)
except socket.error, error:
if error.args[0] in (errno.ENFILE, errno.EMFILE):
# max open connections
greenhouse.pause_for(0.01)
elif error.args[0] == errno.EINVAL:
# server socket was shut down
break
elif error.args[0] == errno.EBADF:
# believe it or not, this is the graceful shutdown
# case. see the comments in shutdown() below
break
raise
else:
# might be a long time before the next accept call returns
greenhouse.schedule(handler.serve_all)
del handler, client_sock
except KeyboardInterrupt:
pass
finally:
self._cleanup()
def test_balance(self):
ch = greenhouse.utils.Channel()
for i in xrange(1, 51):
greenhouse.schedule(ch.send, (None,))
greenhouse.pause()
assert ch.balance == i, (ch.balance, i)
for i in xrange(49, -1, -1):
ch.receive()
assert ch.balance == i, (ch.balance, i)
for i in xrange(-1, -51, -1):
greenhouse.schedule(ch.receive)
greenhouse.pause()
assert ch.balance == i, ch.balance
for i in xrange(-49, 1):
ch.send(None)
assert ch.balance == i, ch.balance
def get_urls_queue(urls, parallelism=None):
in_q = greenhouse.Queue()
out_q = greenhouse.Queue()
results = {}
stop = object()
parallelism = parallelism or len(urls)
for i in xrange(parallelism):
greenhouse.schedule(_queue_runner, args=(in_q, out_q, stop))
for url in urls:
in_q.put(url)
for url in urls:
url, result = out_q.get()
results[url] = result
for i in xrange(parallelism):
in_q.put(stop)
return results
def get_urls_queue(urls, parallelism=None):
in_q = greenhouse.Queue()
out_q = greenhouse.Queue()
results = {}
stop = object()
parallelism = parallelism or len(urls)
for i in xrange(parallelism):
greenhouse.schedule(_queue_runner, args=(in_q, out_q, stop))
for url in urls:
in_q.put(url)
for url in urls:
url, result = out_q.get()
results[url] = result
for i in xrange(parallelism):
in_q.put(stop)
return results
def handle(self):
self.connection.sendall("enter your name up to 20 characters\r\n")
name = self.rfile.readline().rstrip()
if len(name) > 20:
self.connection.sendall("name too long!\r\n")
return
if name in connections:
self.connection.sendall("already have a '%s'\r\n" % name)
return
connections[name] = self
greenhouse.schedule(self._broadcast,
args=("** %s has entered the room" % name, ))
for line in self.rfile:
if not line:
del connections[name]
break
greenhouse.schedule(self._broadcast,
args=("%s: %s" % (name, line.rstrip()), self))
greenhouse.schedule(self._broadcast,
args=("** %s has left the room" % name, self))
def handle(self):
self.connection.sendall("enter your name up to 20 characters\r\n")
name = self.rfile.readline().rstrip()
if len(name) > 20:
self.connection.sendall("name too long!\r\n")
return
if name in connections:
self.connection.sendall("already have a '%s'\r\n" % name)
return
connections[name] = self
greenhouse.schedule(self._broadcast, args=(
"** %s has entered the room" % name,))
for line in self.rfile:
if not line:
del connections[name]
break
greenhouse.schedule(self._broadcast, args=(
"%s: %s" % (name, line.rstrip()), self))
greenhouse.schedule(self._broadcast, args=(
"** %s has left the room" % name, self))
def run(self):
self.running = True
if not self.connected:
self.connect()
if not self.registered:
self.register()
self.join_rooms()
sockfile = self.sock.makefile()
while self.running:
line = sockfile.readline()
if not line:
break
greenhouse.schedule(self._run_one, args=(line, ))
else:
for room in self.in_rooms:
self.cmd("part", room)
self.quit()
def test_recv_with_recver_preference(self):
ch = greenhouse.utils.Channel()
sendcounter = []
def sender():
ch.send(None)
sendcounter.append(None)
for i in xrange(20):
greenhouse.schedule(sender)
greenhouse.pause()
assert ch.balance == 20, ch.balance
assert len(sendcounter) == 0
for i in xrange(20):
ch.receive()
# with recver preference, this doesn't switch to the blocked sender
assert len(sendcounter) == 0
greenhouse.pause() # now the sender greenlets will finish
assert len(sendcounter) == 20
def run(self):
self.running = True
if not self.connected:
self.connect()
if not self.registered:
self.register()
self.join_rooms()
sockfile = self.sock.makefile()
while self.running:
line = sockfile.readline()
if not line:
break
greenhouse.schedule(self._run_one, args=(line,))
else:
for room in self.in_rooms:
self.cmd("part", room)
self.quit()
def abort(self, klass, exc, tb=None):
'''Finish this future in an error state
Takes a standard exception triple as arguments (like those returned by
``sys.exc_info``).
Any :class:`Dependent` that are children of this one will also be
aborted.
:param class klass: the class of the exception
:param Exception exc: the exception instance itself
:param traceback tb: the traceback associated with the exception
:raises:
:class:`AlreadyComplete <zugzug.errors.AlreadyComplete>` if
already complete
'''
if self._done.is_set():
raise errors.AlreadyComplete()
self._failure = (klass, exc, tb)
for eb in self._errbacks:
greenhouse.schedule(eb, args=(klass, exc, tb))
self._errbacks = None
for wait in list(self._waits):
wait.finish(self)
self._waits = None
for child in self._children:
child = child()
if child is None:
continue
child.abort(klass, exc, tb)
self._children = None
self._done.set()
def abort(self, klass, exc, tb=None):
'''Finish this future in an error state
Takes a standard exception triple as arguments (like those returned by
``sys.exc_info``).
Any :class:`Dependent` that are children of this one will also be
aborted.
:param class klass: the class of the exception
:param Exception exc: the exception instance itself
:param traceback tb: the traceback associated with the exception
:raises:
:class:`AlreadyComplete <zugzug.errors.AlreadyComplete>` if
already complete
'''
if self._done.is_set():
raise errors.AlreadyComplete()
self._failure = (klass, exc, tb)
for eb in self._errbacks:
greenhouse.schedule(eb, args=(klass, exc, tb))
self._errbacks = None
for wait in list(self._waits):
wait.finish(self)
self._waits = None
for child in self._children:
child = child()
if child is None:
continue
child.abort(klass, exc, tb)
self._children = None
self._done.set()
def connection_handler(clientsock):
clientsock.sendall("enter your name up to 20 characters\r\n")
name = clientsock.recv(8192).rstrip("\r\n")
if len(name) > 20:
clientsock.close()
return
CONNECTED[name] = clientsock
greenhouse.schedule(broadcast, args=("*** %s has entered\n" % name, name))
sockfile = clientsock.makefile("r")
while 1:
line = sockfile.readline()
if not line:
CONNECTED.pop(name)
break
greenhouse.schedule(broadcast, args=(
"%s: %s\n" % (name, line.rstrip("\r\n")), name))
broadcast("*** %s has left the building\n" % name)
def test_iteration(self):
def runner(x):
if x > 10:
pool.close()
return x**2
def putter(pool):
i = 0
while 1:
i += 1
pool.put(i)
greenhouse.pause()
pool = self.POOL(runner, 3)
pool.start()
greenhouse.schedule(putter, args=(pool, ))
results = []
for item in pool:
results.append(item)
self.assertEqual(results, [x**2 for x in xrange(1, 11)])
def test_iteration(self):
def runner(x):
if x > 10:
pool.close()
return x ** 2
def putter(pool):
i = 0
while 1:
i += 1
pool.put(i)
greenhouse.pause()
pool = self.POOL(runner, 3)
pool.start()
greenhouse.schedule(putter, args=(pool,))
results = []
for item in pool:
results.append(item)
self.assertEqual(results, [x ** 2 for x in xrange(1, 11)])
def connection_handler(clientsock):
clientsock.sendall("enter your name up to 20 characters\r\n")
name = clientsock.recv(8192).rstrip("\r\n")
if len(name) > 20:
clientsock.close()
return
CONNECTED[name] = clientsock
greenhouse.schedule(broadcast, args=("*** %s has entered\n" % name, name))
sockfile = clientsock.makefile("r")
while 1:
line = sockfile.readline()
if not line:
CONNECTED.pop(name)
break
greenhouse.schedule(broadcast,
args=("%s: %s\n" % (name, line.rstrip("\r\n")),
name))
broadcast("*** %s has left the building\n" % name)
def eval(tup, func):
for tup in _findall(tup):
greenhouse.schedule(func, args=(tup,))
def eval(tup, func):
for tup in _findall(tup):
greenhouse.schedule(func, args=(tup, ))
def spawn(self, after=None):
if after is None:
schedule(self.greenlet)
else:
schedule_in(after, self.greenlet)
return self
def spawn(self, after=None):
if after is None:
schedule(self.greenlet)
else:
schedule_in(after, self.greenlet)
return self
def _background(f):
greenhouse.schedule(f)
