def test_queue_multi_consumer(): q = Queue() c = 10000 s1 = [0] s2 = [0] def g(seen): def run(): for x in xrange(c): v = q.get() seen[0] += 1 sleep() return run def p(): for x in xrange(c): q.put(x) sleep(1) print s1, s2 WVPASS(s1[0] < c) WVPASS(s2[0] < c) WVPASS(s1[0] + s2[0] == c) quickstop() quickstart(p, g(s1), g(s2))
def run(self, nameservers, *objs): nameservers = [(h, int(p)) for h, p in (i.split(':') for i in nameservers)] runem = [] if self.run_nameserver: runem.append( Thunk(lambda: run_consensus_server(self.run_nameserver, nameservers))) runem.append(self) handler_functions = dict( (v, k) for k, v in self.local_handlers.iteritems()) final_o = [] for o in objs: if type(o.__class__) is ConvoyRegistrar: r = o.__class__ self.roles_wanted.add(r) for m in self.role_messages[r]: assert m not in self.local_handlers, \ "cannot add two instances for same role/message" self.local_handlers[m] = \ getattr(o, 'handle_' + m) else: final_o.append(o) self.ns = ConvoyNameService(nameservers) runem.append(self.ns) runem.append(self.deliver) runem.extend(final_o) runem.append(ConvoyService()) quickstart(*runem)
def run(self, nameservers, *objs): nameservers = [(h, int(p)) for h, p in (i.split(':') for i in nameservers)] runem = [] if self.run_nameserver: runem.append( Thunk(lambda: run_consensus_server(self.run_nameserver, nameservers))) runem.append(self) handler_functions = dict((v, k) for k, v in self.local_handlers.iteritems()) final_o = [] for o in objs: if type(o.__class__) is ConvoyRegistrar: r = o.__class__ self.roles_wanted.add(r) for m in self.role_messages[r]: assert m not in self.local_handlers, \ "cannot add two instances for same role/message" self.local_handlers[m] = \ getattr(o, 'handle_' + m) else: final_o.append(o) self.ns = ConvoyNameService(nameservers) runem.append(self.ns) runem.append(self.deliver) runem.extend(final_o) runem.append(ConvoyService()) quickstart(*runem)
def test_fork_child_exception(): def parent(): fork_child(dependent_child) sleep(0.1) a = b # undef quickstart(parent)
def test_fork_child_normal_death(): def parent(): fork_child(dependent_child) sleep(0.1) # implied, I end.. quickstart(parent)
def main(): parser = optparse.OptionParser("Usage: %prog PID") parser.add_option( '-p', '--port', default=port, type="int", help="The port to listen on for console connections", ) options, args = parser.parse_args() if not args: parser.print_usage() raise SystemExit(1) if args[0] == 'dummy': print "PID", os.getpid() def wait_for_signal(): log = diesel.log.name('dummy') log.min_level = diesel.loglevels.INFO install_console_signal_handler() while True: log.info("sleeping") diesel.sleep(5) diesel.quickstart(wait_for_signal) else: pid = int(args[0]) svc = diesel.Service(console_for(pid), options.port) diesel.quickstart(svc)
def test_fork_child_exception(): def parent(): fork_child(dependent_child) sleep(0.1) a = b # undef quickstart(parent)
def test_fork_child_normal_death(): def parent(): fork_child(dependent_child) sleep(0.1) # implied, I end.. quickstart(parent)
def test_basic_fork(): def parent(): v = [0] fork(tottering_child, v) sleep(0.1) WVPASS(v[0] == 1) quickstop() quickstart(parent)
def test_basic_fork(): def parent(): v = [0] fork(tottering_child, v) sleep(0.1) WVPASS(v[0] == 1) quickstop() quickstart(parent)
def test_fork_many(): def parent(): COUNT = 10000 v = [0] for x in xrange(COUNT): fork(tottering_child, v) sleep(8) # long enough on core 2-era WVPASS(v[0] == COUNT) quickstop() quickstart(parent)
def test_basic_sleep(): def l(): t = time() sleep(0.3) delt = time() - t print delt WVPASS(delt > 0.275 and delt < 0.325) quickstop() quickstart(l)
def test_fork_many(): def parent(): COUNT = 10000 v = [0] for x in xrange(COUNT): fork(tottering_child, v) sleep(8) # long enough on core 2-era WVPASS(v[0] == COUNT) quickstop() quickstart(parent)
def test_fire_miss(): v = [0] def w(): while True: wait("boom!") v[0] += 1 def f(): sleep(0.05) fire("fizz!") sleep(0.05) WVPASS(v[0] == 0) # should not have woken up! quickstop() quickstart(f, w, w)
def test_sleep_independence(): v = [0] def i(): v[0] += 1 sleep(0.1) v[0] += 1 def l(): t = time() sleep(0.05) WVPASS(v[0] == 2) sleep(0.1) WVPASS(v[0] == 4) quickstop() quickstart(l, i, i)
def test_queue_put_noloss(): q = Queue() c = 10000 done = [0] def g(): for x in xrange(c): v = q.get() WVPASS(v == x) done[0] = 1 def p(): for x in xrange(c): q.put(x) sleep(1) WVPASS(done[0] == 1) quickstop() quickstart(p, g)
def test_fire_multiple(): v = [0] def w(): while True: wait("boom!") v[0] += 1 def f(): sleep(0.05) fire("boom!") sleep(0.05) WVPASS(v[0] == 2) sleep(0.05) fire("boom!") sleep(0.05) WVPASS(v[0] == 4) quickstop() quickstart(f, w, w)
def test_basic_fire(): v = [0] def w(): while True: wait("boom!") v[0] += 1 def f(): sleep(0.05) fire("boom!") sleep(0.05) WVPASS(v[0] == 1) sleep(0.05) fire("boom!") sleep(0.05) WVPASS(v[0] == 2) quickstop() quickstart(f, w)
def test_sleep_zero(): """Sleep w/out argument allows other loops to run """ v = [0] def i(): x = 0 while True: v[0] += 1 sleep() def l(): sleep(0.05) cur = v[0] sleep() now = v[0] WVPASS(now == cur + 1) quickstop() quickstart(l, i)
def test_queue_put_noloss(): q = Queue() c = 10000 done = [0] def g(): for x in xrange(c): v = q.get() WVPASS(v == x) done[0] = 1 def p(): for x in xrange(c): q.put(x) sleep(1) WVPASS(done[0] == 1) quickstop() quickstart(p, g)
def test_queue_multi_consumer(): q = Queue() c = 10000 s1 = [0] s2 = [0] def g(seen): def run(): for x in xrange(c): v = q.get() seen[0] += 1 sleep() return run def p(): for x in xrange(c): q.put(x) sleep(1) print s1, s2 WVPASS(s1[0] < c) WVPASS(s2[0] < c) WVPASS(s1[0] + s2[0] == c) quickstop() quickstart(p, g(s1), g(s2))
def main(): parser = optparse.OptionParser("Usage: %prog PID") parser.add_option( '-p', '--port', default=port, type="int", help="The port to listen on for console connections", ) options, args = parser.parse_args() if not args: parser.print_usage() raise SystemExit(1) if args[0] == 'dummy': print "PID", os.getpid() def wait_for_signal(): log = diesel.log.name('dummy') log.min_level = diesel.loglevels.INFO install_console_signal_handler() while True: log.info("sleeping") diesel.sleep(5) diesel.quickstart(wait_for_signal) else: pid = int(args[0]) svc = diesel.Service(console_for(pid), options.port) diesel.quickstart(svc)
from diesel import quickstart, quickstop, sleep, first from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq import time def get_messages(): outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), bind="tcp://127.0.0.1:5000") for x in xrange(1000): t, m = first(sleep=1.0, waits=[outsock]) if t == 'sleep': print "sleep timeout!" else: print "zmq:", m quickstop() quickstart(get_messages)
#!/usr/bin/env python # encoding: utf-8 from diesel import quickstart, quickstop, sleep from diesel.util.event import Countdown # This "done" event won't be set until it is ticked 3 times. done = Countdown(3) def main(): for i in range(3): print "Tick ..." sleep(1) done.tick() def stop_when_done(): done.wait() print "Boom!" quickstop() quickstart(main, stop_when_done)
print "First, without dreadlock" t = time.time() cd = Countdown(500) for x in xrange(500): fork(nolocks, cd) cd.wait() print "Test took %.3f" % (time.time() - t) print "%s < %s?" % (v, 50 * 500) assert (v < 50 * 500) print '-' * 72 v = 0 locker = DreadlockService('localhost', 6001, 100) t = time.time() cd = Countdown(500) for x in xrange(500): fork(withlocks, locker, cd) cd.wait() print "Test took %.3f" % (time.time() - t) print "%s == %s?" % (v, 50 * 500) assert (v == 50 * 500) print '-' * 72 print "okay." quickstart(runner)
def ipython(): if not IPYTHON_AVAILABLE: print >> sys.stderr, "IPython not found." raise SystemExit(1) diesel.quickstart(interact_ipython)
sleep_pool = ProcessPool(2, sleep_and_return) def main(): def waiting(ident): print ident, "waiting ..." t = sleep_pool(4) print ident, "woken up after", t diesel.fork(waiting, 'a') diesel.fork(waiting, 'b') diesel.fork(waiting, 'c') for i in xrange(11): print "busy!" diesel.sleep(1) div = spawn(lambda x, y: x / y) try: div(1, 0) except ZeroDivisionError, e: diesel.log.error(e.original_traceback) print '^^ That was an intentional exception.' term(div) psleep = spawn(sleep_and_return) diesel.fork(psleep, 0.5) diesel.fork(psleep, 0.5) diesel.sleep(1) print '^^ That was an intentional exception.' diesel.quickstop() diesel.quickstart(sleep_pool.pool, main)
def append_stats(): rawstats = subprocess.Popen(['ps -p %d -f' % pid], shell=True, stdout=subprocess.PIPE).communicate()[0] header, data = rawstats.split('\n', 1) procstats = [d for d in data.split(' ') if d] cpustats.append(int(procstats[3])) while True: diesel.sleep(1) diesel.thread(append_stats) def main(): diesel.fork(track_cpu_stats) actor_pairs = int(sys.argv[1]) done = Countdown(actor_pairs) for i in xrange(actor_pairs): pair(done) start = time.time() done.wait() print "done in %.2f secs" % (time.time() - start) diesel.sleep(1) diesel.quickstop() if __name__ == '__main__': diesel.set_log_level(diesel.loglevels.ERROR) diesel.quickstart(main) print cpustats
#!/usr/bin/env python # encoding: utf-8 import diesel def holla_back(): while True: message = diesel.receive(diesel.datagram) shouted_message = message.upper() diesel.send(shouted_message) diesel.quickstart(diesel.UDPService(holla_back, 1234))
from diesel import quickstart, sleep from diesel.util.pool import ThreadPool from diesel.protocols.http import HttpClient, HttpHeaders import random def handle_it(i): print 'S', i sleep(random.random()) print 'E', i def c(): for x in xrange(0, 20): yield x make_it = c().next threads = ThreadPool(10, handle_it, make_it) quickstart(threads)
def start_service(iface, port, whois): """ Start the service. """ diesel.quickstart( diesel.Service(functools.partial(respond, whois), port, iface))
import diesel from diesel.protocols.zeromq import ( DieselZMQService, zmq, zctx, DieselZMQSocket, ) NUM_CLIENTS = 100 cids = range(NUM_CLIENTS) def echo_client(): sock = zctx.socket(zmq.DEALER) sock.identity = "client:%d" % cids.pop() s = DieselZMQSocket(sock, connect='tcp://127.0.0.1:4321') for i in xrange(10): s.send('msg:%d' % i) r = s.recv() assert r == 'msg:%d' % i print sock.identity, 'received', r class EchoService(DieselZMQService): def handle_client_packet(self, packet, ctx): return packet echo_svc = EchoService('tcp://*:4321') diesel.quickstart(echo_svc.run, *(echo_client for i in xrange(NUM_CLIENTS)))
def handle_echo(remote_addr): while True: message = until('\r\n') send("you said: %s" % message) class EchoClient(Client): @call def echo(self, message): send(message + '\r\n') back = until("\r\n") return back log = log.name('echo-system') def do_echos(): with EchoClient('localhost', 8000, ssl_ctx=SSL.Context(SSL.TLSv1_METHOD)) as client: t = time.time() for x in xrange(5000): msg = "hello, world #%s!" % x echo_result = client.echo(msg) assert echo_result.strip() == "you said: %s" % msg log.info('5000 loops in {0:.2f}s', time.time() - t) quickstop() quickstart(Service(handle_echo, port=8000, ssl_ctx=server_ctx), do_echos)
'''Example of event firing. ''' import time import random from diesel import (quickstart, quickstop, sleep, fire, wait, log, loglevels, set_log_level) set_log_level(loglevels.DEBUG) def gunner(): x = 1 while True: fire('bam', x) x += 1 sleep() def sieged(): t = time.time() while True: n = wait('bam') if n % 10000 == 0: log.info(str(n)) if n == 50000: delt = time.time() - t log.debug("50,000 messages in {0:.3f}s {1:.1f}/s)", delt, 50000 / delt) quickstop() log = log.name('fire-system') quickstart(gunner, sieged)
import random from diesel import (quickstart, quickstop, sleep, fire, wait, log, loglevels, set_log_level) set_log_level(loglevels.DEBUG) def gunner(): x = 1 while True: fire('bam', x) x += 1 sleep() def sieged(): t = time.time() while True: n = wait('bam') if n % 10000 == 0: log.info(str(n)) if n == 50000: delt = time.time() - t log.debug("50,000 messages in {0:.3f}s {1:.1f}/s)", delt, 50000 / delt) quickstop() log = log.name('fire-system') quickstart(gunner, sieged)
from diesel import quickstart, quickstop, sleep from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq import time def send_message(): outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), connect="tcp://127.0.0.1:5000") for x in xrange(500000): outsock.send("yo dawg %s" % x) if x % 1000 == 0: sleep() def tick(): while True: print "Other diesel stuff" sleep(1) quickstart(send_message, tick)
# vim:ts=4:sw=4:expandtab '''Example of deferring blocking calls to threads ''' from diesel import log, thread, quickstart import time from functools import partial def blocker(taskid, sleep_time): while True: def f(): time.sleep(sleep_time) thread(f) log.info('yo! {0} from {1} task', time.time(), taskid) quickstart(partial(blocker, 'fast', 1), partial(blocker, 'slow', 5))
cd = Countdown(4) def putter(): log = glog.sublog("putter", glog.info) log.info("putting 100000 things on log") for x in xrange(100000): q.put(x) sleep() def getter(): log = glog.sublog("getter", glog.info) got = 0 while got < 25000: try: s = q.get(timeout=3) sleep() except QueueTimeout: log.warn("timeout before getting a value, retrying...") continue got += 1 log.info("SUCCESS! got all 25,000") cd.tick() def manage(): cd.wait() quickstop() quickstart(manage, putter, [getter for x in xrange(4)])
def worker(): num_processed = 0 while True: val = diesel.wait(q) if val == shutdown: break num_processed += 1 fmt_args = (diesel.core.current_loop, num_processed) print "%s, worker done (processed %d items)" % fmt_args dones.put('done') def main(): start = time.time() for i in xrange(NUM_ITEMS): q.put('item %d' % i) for i in xrange(NUM_WORKERS): q.put(shutdown) for i in xrange(NUM_WORKERS): diesel.fork_child(worker) for i in xrange(NUM_WORKERS): dones.get() print 'all workers done in %.2f secs' % (time.time() - start) diesel.quickstop() if __name__ == '__main__': diesel.quickstart(main)
else: cls = ConvoySetTimeout if rest: clock = rest[0] values = rest[1:] return cls(ConsensusSet(values, clock)) else: return cls() else: assert result == "SET-OKAY" clock = rest[0] values = rest[1:] return ConsensusSet(values, clock) @call def keep_alive(self): send("KEEPALIVE\r\n") assert until_eol().strip().upper() == "KEEPALIVE-OKAY" cargo = ConvoyNameService([('localhost', 1111), ('localhost', 1112), ('localhost', 1113)]) if __name__ == '__main__': def run(): print cargo.clear("foo") print cargo.set("foo", "bar") print cargo.lookup("foo") quickstop() from diesel import quickstart, quickstop quickstart(cargo, run)
if __name__ == '__main__': def f(): with PostgreSQLClient(database="test", user="******") as client: with client.transact: client.execute("INSERT INTO companies (name) values ($1)" , "JamieCo3") client.execute("INSERT INTO typtest values ($1, $2, $3, $4, $5, $6)" , "string", "string", 14, 155, 23923, True) #for row in client.query("SELECT * FROM companies", buffer=500): # print row.name # print row.id for row in client.query("SELECT * FROM typtest", buffer=500): print row client.execute("UPDATE companies set name = $1 where id < $2", "marky co", 5) print '\n\n~~~done!~~~' def g(): with PostgreSQLClient(database="test", user="******") as client: for x in xrange(500): r = client.query_one( "select * from counters where id = $1 for update", 1) print 'done' from diesel import quickstart quickstart(f, g, g, g, g, g, g, g, g, g, g, g)
LISTENERS = 10 EVENTS = 5 cd = Countdown(LISTENERS * EVENTS) f = Fanout() def listener(x): with f.sub() as q: while True: v = q.get() print '%s <- %s' % (x, v) cd.tick() def teller(): for x in xrange(EVENTS): sleep(2) f.pub(x) def killer(): cd.wait() quickstop() from functools import partial quickstart(killer, teller, *[partial(listener, x) for x in xrange(LISTENERS)])
if __name__ == '__main__': def f(): with PostgreSQLClient(database="test", user="******") as client: with client.transact: client.execute("INSERT INTO companies (name) values ($1)", "JamieCo3") client.execute( "INSERT INTO typtest values ($1, $2, $3, $4, $5, $6)", "string", "string", 14, 155, 23923, True) #for row in client.query("SELECT * FROM companies", buffer=500): # print row.name # print row.id for row in client.query("SELECT * FROM typtest", buffer=500): print row client.execute("UPDATE companies set name = $1 where id < $2", "marky co", 5) print '\n\n~~~done!~~~' def g(): with PostgreSQLClient(database="test", user="******") as client: for x in xrange(500): r = client.query_one( "select * from counters where id = $1 for update", 1) print 'done' from diesel import quickstart quickstart(f, g, g, g, g, g, g, g, g, g, g, g)
from diesel import quickstart, quickstop, sleep, first from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq import time def get_messages(): outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), bind="tcp://127.0.0.1:5000") for x in xrange(1000): t, m = first(sleep=1.0, waits=[outsock]) if t == 'sleep': print "sleep timeout!" else: print "zmq:", m quickstop() quickstart(get_messages)
def python(): diesel.quickstart(interact_python)
def python(): diesel.quickstart(interact_python)
def ipython(): if not IPYTHON_AVAILABLE: print >> sys.stderr, "IPython not found." raise SystemExit(1) diesel.quickstart(interact_ipython)
#!/usr/bin/env python # encoding: utf-8 from diesel import quickstart, sleep, quickstop from diesel.util.event import Event def coordinator(): # Pretend to do something ... sleep(3) # Done, fire the event. print "Coordinator done." ev.set() def consumer(): print "Waiting ..." ev.wait() print "The event was triggered!" def late_consumer(): sleep(4) consumer() quickstop() ev = Event() quickstart(coordinator, consumer, consumer, consumer, late_consumer)
if line == "": break parts = line.split() cmd = parts[0] if cmd == "get": key = parts[1] try: val = CACHE[key] send("VALUE %s 0 %d\r\n" % (key, len(val))) send(val + "\r\n") except KeyError: pass send("END\r\n") elif cmd == "set": key = parts[1] #exp = parts[2] #flags = parts[3] length = int(parts[4]) val = receive(length + 2)[:length] CACHE[key] = val send("STORED\r\n") if __name__ == "__main__": quickstart(Service(handle_con, 11211))
'''Test for client to dreadlock network lock service. ''' import uuid from diesel import sleep, quickstart from diesel.protocols.dreadlock import DreadlockService locker = DreadlockService('localhost', 6001) def f(): with locker.hold("foo", 30): id = uuid.uuid4() print "start!", id sleep(2) print "end!", id quickstart(f, f, f, f, f)
assert isinstance(there, Point) # resolve on put p5.put('there', Point(1, 9)) # should resolve b/c safe=True assert len(c.get('testing.pickles', 'there')['content']) == 1 # Doing stuff with different client ids but the same vector clock. c.set_client_id('diff 1') assert b.put('diff', '---') c.set_client_id('diff 2') assert b.put('diff', '+++') assert b.get('diff') == '+++' # Provoking an error try: # Tell Riak to require 10000 nodes to write this before success. c.put('testing', 'error!', 'oh noes!', w=10000) except RiakErrorResp, e: assert e.errcode == 1, e.errcode assert e.errmsg == '{n_val_violation,3}', e.errmsg assert repr(e) == "RiakErrorResp: {n_val_violation,3}", repr(e) except Exception, e: assert 0, "UNEXPECTED EXCEPTION: %r" % e else: assert 0, "DID NOT RAISE" diesel.quickstop() diesel.quickstart(test_client) del Point
import sys from diesel import quickstart, fork_from_thread from diesel.util.queue import Queue from thread import start_new_thread q = Queue() def consume(): while True: v = q.get() print 'DIESEL GOT', v def put(line): q.put(line) def create(): while True: line = sys.stdin.readline() print 'iter!', line fork_from_thread(put, line) start_new_thread(create, ()) quickstart(consume)
from diesel import quickstart, quickstop, sleep from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq import time def handle_messages(): insock = DieselZMQSocket(zctx.socket(zmq.DEALER), bind="inproc://foo") for x in xrange(500000): msg = insock.recv() assert msg == "yo dawg %s" % x delt = time.time() - t print "500000 messages in %ss (%.1f/s)" % (delt, 500000.0 / delt) quickstop() def send_message(): global t outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), connect="inproc://foo") t = time.time() for x in xrange(500000): outsock.send("yo dawg %s" % x) if x % 1000 == 0: sleep() def tick(): while True: print "Other diesel stuff" sleep(1) quickstart(handle_messages, send_message, tick)
from diesel import quickstart, quickstop from montageclient import MontageClient from user_palm import UserInfo # run with montage/examples/basic_proxy.hs def test_resolution(cl, bucket, key): [data_1, data_2, data_3] = [ UserInfo(uid=i) for i in xrange(1,4) ] first = cl.put(bucket, key, data_1.dumps()) second = cl.put(bucket, key, data_2.dumps()) third = cl.put(bucket, key, data_3.dumps()) resolved = cl.get(bucket, key) assert UserInfo(resolved.data) == data_3, "Failed last write wins resolution" print "\nsuccessfully did last write wins resolution\n" def main(): cl = MontageClient('localhost', 7078) test_resolution(cl, 'u-info', '1') quickstop() quickstart(main)
'''Test for client to dreadlock network lock service. ''' import uuid from diesel import sleep, quickstart from diesel.protocols.dreadlock import DreadlockService locker = DreadlockService('localhost', 6001) def f(): with locker.hold("foo", 30): id = uuid.uuid4() print "start!", id sleep(2) print "end!", id quickstart(f, f, f, f, f)