def test_queue_multi_consumer(): q = Queue() c = 10000 s1 = [0] s2 = [0] def g(seen): def run(): for x in xrange(c): v = q.get() seen[0] += 1 sleep() return run def p(): for x in xrange(c): q.put(x) sleep(1) print s1, s2 WVPASS(s1[0] < c) WVPASS(s2[0] < c) WVPASS(s1[0] + s2[0] == c) quickstop() quickstart(p, g(s1), g(s2))
def run(self, nameservers, *objs): nameservers = [(h, int(p)) for h, p in (i.split(':') for i in nameservers)] runem = [] if self.run_nameserver: runem.append( Thunk(lambda: run_consensus_server(self.run_nameserver, nameservers))) runem.append(self) handler_functions = dict( (v, k) for k, v in self.local_handlers.iteritems()) final_o = [] for o in objs: if type(o.__class__) is ConvoyRegistrar: r = o.__class__ self.roles_wanted.add(r) for m in self.role_messages[r]: assert m not in self.local_handlers, \ "cannot add two instances for same role/message" self.local_handlers[m] = \ getattr(o, 'handle_' + m) else: final_o.append(o) self.ns = ConvoyNameService(nameservers) runem.append(self.ns) runem.append(self.deliver) runem.extend(final_o) runem.append(ConvoyService()) quickstart(*runem)
def run(self, nameservers, *objs): nameservers = [(h, int(p)) for h, p in (i.split(':') for i in nameservers)] runem = [] if self.run_nameserver: runem.append( Thunk(lambda: run_consensus_server(self.run_nameserver, nameservers))) runem.append(self) handler_functions = dict((v, k) for k, v in self.local_handlers.iteritems()) final_o = [] for o in objs: if type(o.__class__) is ConvoyRegistrar: r = o.__class__ self.roles_wanted.add(r) for m in self.role_messages[r]: assert m not in self.local_handlers, \ "cannot add two instances for same role/message" self.local_handlers[m] = \ getattr(o, 'handle_' + m) else: final_o.append(o) self.ns = ConvoyNameService(nameservers) runem.append(self.ns) runem.append(self.deliver) runem.extend(final_o) runem.append(ConvoyService()) quickstart(*runem)
def test_fork_child_exception(): def parent(): fork_child(dependent_child) sleep(0.1) a = b # undef quickstart(parent)
def test_fork_child_normal_death(): def parent(): fork_child(dependent_child) sleep(0.1) # implied, I end.. quickstart(parent)
def main(): parser = optparse.OptionParser("Usage: %prog PID") parser.add_option( '-p', '--port', default=port, type="int", help="The port to listen on for console connections", ) options, args = parser.parse_args() if not args: parser.print_usage() raise SystemExit(1) if args[0] == 'dummy': print "PID", os.getpid() def wait_for_signal(): log = diesel.log.name('dummy') log.min_level = diesel.loglevels.INFO install_console_signal_handler() while True: log.info("sleeping") diesel.sleep(5) diesel.quickstart(wait_for_signal) else: pid = int(args[0]) svc = diesel.Service(console_for(pid), options.port) diesel.quickstart(svc)
def test_basic_fork(): def parent(): v = [0] fork(tottering_child, v) sleep(0.1) WVPASS(v[0] == 1) quickstop() quickstart(parent)
def test_fork_many(): def parent(): COUNT = 10000 v = [0] for x in xrange(COUNT): fork(tottering_child, v) sleep(8) # long enough on core 2-era WVPASS(v[0] == COUNT) quickstop() quickstart(parent)
def test_basic_sleep(): def l(): t = time() sleep(0.3) delt = time() - t print delt WVPASS(delt > 0.275 and delt < 0.325) quickstop() quickstart(l)
def test_fire_miss(): v = [0] def w(): while True: wait("boom!") v[0] += 1 def f(): sleep(0.05) fire("fizz!") sleep(0.05) WVPASS(v[0] == 0) # should not have woken up! quickstop() quickstart(f, w, w)
def test_sleep_independence(): v = [0] def i(): v[0] += 1 sleep(0.1) v[0] += 1 def l(): t = time() sleep(0.05) WVPASS(v[0] == 2) sleep(0.1) WVPASS(v[0] == 4) quickstop() quickstart(l, i, i)
def test_queue_put_noloss(): q = Queue() c = 10000 done = [0] def g(): for x in xrange(c): v = q.get() WVPASS(v == x) done[0] = 1 def p(): for x in xrange(c): q.put(x) sleep(1) WVPASS(done[0] == 1) quickstop() quickstart(p, g)
def test_fire_multiple(): v = [0] def w(): while True: wait("boom!") v[0] += 1 def f(): sleep(0.05) fire("boom!") sleep(0.05) WVPASS(v[0] == 2) sleep(0.05) fire("boom!") sleep(0.05) WVPASS(v[0] == 4) quickstop() quickstart(f, w, w)
def test_basic_fire(): v = [0] def w(): while True: wait("boom!") v[0] += 1 def f(): sleep(0.05) fire("boom!") sleep(0.05) WVPASS(v[0] == 1) sleep(0.05) fire("boom!") sleep(0.05) WVPASS(v[0] == 2) quickstop() quickstart(f, w)
def test_sleep_zero(): """Sleep w/out argument allows other loops to run """ v = [0] def i(): x = 0 while True: v[0] += 1 sleep() def l(): sleep(0.05) cur = v[0] sleep() now = v[0] WVPASS(now == cur + 1) quickstop() quickstart(l, i)
from diesel import quickstart, quickstop, sleep, first from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq import time def get_messages(): outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), bind="tcp://127.0.0.1:5000") for x in xrange(1000): t, m = first(sleep=1.0, waits=[outsock]) if t == 'sleep': print "sleep timeout!" else: print "zmq:", m quickstop() quickstart(get_messages)
#!/usr/bin/env python # encoding: utf-8 from diesel import quickstart, quickstop, sleep from diesel.util.event import Countdown # This "done" event won't be set until it is ticked 3 times. done = Countdown(3) def main(): for i in range(3): print "Tick ..." sleep(1) done.tick() def stop_when_done(): done.wait() print "Boom!" quickstop() quickstart(main, stop_when_done)
print "First, without dreadlock" t = time.time() cd = Countdown(500) for x in xrange(500): fork(nolocks, cd) cd.wait() print "Test took %.3f" % (time.time() - t) print "%s < %s?" % (v, 50 * 500) assert (v < 50 * 500) print '-' * 72 v = 0 locker = DreadlockService('localhost', 6001, 100) t = time.time() cd = Countdown(500) for x in xrange(500): fork(withlocks, locker, cd) cd.wait() print "Test took %.3f" % (time.time() - t) print "%s == %s?" % (v, 50 * 500) assert (v == 50 * 500) print '-' * 72 print "okay." quickstart(runner)
def ipython(): if not IPYTHON_AVAILABLE: print >> sys.stderr, "IPython not found." raise SystemExit(1) diesel.quickstart(interact_ipython)
sleep_pool = ProcessPool(2, sleep_and_return) def main(): def waiting(ident): print ident, "waiting ..." t = sleep_pool(4) print ident, "woken up after", t diesel.fork(waiting, 'a') diesel.fork(waiting, 'b') diesel.fork(waiting, 'c') for i in xrange(11): print "busy!" diesel.sleep(1) div = spawn(lambda x, y: x / y) try: div(1, 0) except ZeroDivisionError, e: diesel.log.error(e.original_traceback) print '^^ That was an intentional exception.' term(div) psleep = spawn(sleep_and_return) diesel.fork(psleep, 0.5) diesel.fork(psleep, 0.5) diesel.sleep(1) print '^^ That was an intentional exception.' diesel.quickstop() diesel.quickstart(sleep_pool.pool, main)
def append_stats(): rawstats = subprocess.Popen(['ps -p %d -f' % pid], shell=True, stdout=subprocess.PIPE).communicate()[0] header, data = rawstats.split('\n', 1) procstats = [d for d in data.split(' ') if d] cpustats.append(int(procstats[3])) while True: diesel.sleep(1) diesel.thread(append_stats) def main(): diesel.fork(track_cpu_stats) actor_pairs = int(sys.argv[1]) done = Countdown(actor_pairs) for i in xrange(actor_pairs): pair(done) start = time.time() done.wait() print "done in %.2f secs" % (time.time() - start) diesel.sleep(1) diesel.quickstop() if __name__ == '__main__': diesel.set_log_level(diesel.loglevels.ERROR) diesel.quickstart(main) print cpustats
#!/usr/bin/env python # encoding: utf-8 import diesel def holla_back(): while True: message = diesel.receive(diesel.datagram) shouted_message = message.upper() diesel.send(shouted_message) diesel.quickstart(diesel.UDPService(holla_back, 1234))
from diesel import quickstart, sleep from diesel.util.pool import ThreadPool from diesel.protocols.http import HttpClient, HttpHeaders import random def handle_it(i): print 'S', i sleep(random.random()) print 'E', i def c(): for x in xrange(0, 20): yield x make_it = c().next threads = ThreadPool(10, handle_it, make_it) quickstart(threads)
def start_service(iface, port, whois): """ Start the service. """ diesel.quickstart( diesel.Service(functools.partial(respond, whois), port, iface))
import diesel from diesel.protocols.zeromq import ( DieselZMQService, zmq, zctx, DieselZMQSocket, ) NUM_CLIENTS = 100 cids = range(NUM_CLIENTS) def echo_client(): sock = zctx.socket(zmq.DEALER) sock.identity = "client:%d" % cids.pop() s = DieselZMQSocket(sock, connect='tcp://127.0.0.1:4321') for i in xrange(10): s.send('msg:%d' % i) r = s.recv() assert r == 'msg:%d' % i print sock.identity, 'received', r class EchoService(DieselZMQService): def handle_client_packet(self, packet, ctx): return packet echo_svc = EchoService('tcp://*:4321') diesel.quickstart(echo_svc.run, *(echo_client for i in xrange(NUM_CLIENTS)))
def handle_echo(remote_addr): while True: message = until('\r\n') send("you said: %s" % message) class EchoClient(Client): @call def echo(self, message): send(message + '\r\n') back = until("\r\n") return back log = log.name('echo-system') def do_echos(): with EchoClient('localhost', 8000, ssl_ctx=SSL.Context(SSL.TLSv1_METHOD)) as client: t = time.time() for x in xrange(5000): msg = "hello, world #%s!" % x echo_result = client.echo(msg) assert echo_result.strip() == "you said: %s" % msg log.info('5000 loops in {0:.2f}s', time.time() - t) quickstop() quickstart(Service(handle_echo, port=8000, ssl_ctx=server_ctx), do_echos)
'''Example of event firing. ''' import time import random from diesel import (quickstart, quickstop, sleep, fire, wait, log, loglevels, set_log_level) set_log_level(loglevels.DEBUG) def gunner(): x = 1 while True: fire('bam', x) x += 1 sleep() def sieged(): t = time.time() while True: n = wait('bam') if n % 10000 == 0: log.info(str(n)) if n == 50000: delt = time.time() - t log.debug("50,000 messages in {0:.3f}s {1:.1f}/s)", delt, 50000 / delt) quickstop() log = log.name('fire-system') quickstart(gunner, sieged)
import random from diesel import (quickstart, quickstop, sleep, fire, wait, log, loglevels, set_log_level) set_log_level(loglevels.DEBUG) def gunner(): x = 1 while True: fire('bam', x) x += 1 sleep() def sieged(): t = time.time() while True: n = wait('bam') if n % 10000 == 0: log.info(str(n)) if n == 50000: delt = time.time() - t log.debug("50,000 messages in {0:.3f}s {1:.1f}/s)", delt, 50000 / delt) quickstop() log = log.name('fire-system') quickstart(gunner, sieged)
from diesel import quickstart, quickstop, sleep from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq import time def send_message(): outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), connect="tcp://127.0.0.1:5000") for x in xrange(500000): outsock.send("yo dawg %s" % x) if x % 1000 == 0: sleep() def tick(): while True: print "Other diesel stuff" sleep(1) quickstart(send_message, tick)
# vim:ts=4:sw=4:expandtab '''Example of deferring blocking calls to threads ''' from diesel import log, thread, quickstart import time from functools import partial def blocker(taskid, sleep_time): while True: def f(): time.sleep(sleep_time) thread(f) log.info('yo! {0} from {1} task', time.time(), taskid) quickstart(partial(blocker, 'fast', 1), partial(blocker, 'slow', 5))
cd = Countdown(4) def putter(): log = glog.sublog("putter", glog.info) log.info("putting 100000 things on log") for x in xrange(100000): q.put(x) sleep() def getter(): log = glog.sublog("getter", glog.info) got = 0 while got < 25000: try: s = q.get(timeout=3) sleep() except QueueTimeout: log.warn("timeout before getting a value, retrying...") continue got += 1 log.info("SUCCESS! got all 25,000") cd.tick() def manage(): cd.wait() quickstop() quickstart(manage, putter, [getter for x in xrange(4)])
def worker(): num_processed = 0 while True: val = diesel.wait(q) if val == shutdown: break num_processed += 1 fmt_args = (diesel.core.current_loop, num_processed) print "%s, worker done (processed %d items)" % fmt_args dones.put('done') def main(): start = time.time() for i in xrange(NUM_ITEMS): q.put('item %d' % i) for i in xrange(NUM_WORKERS): q.put(shutdown) for i in xrange(NUM_WORKERS): diesel.fork_child(worker) for i in xrange(NUM_WORKERS): dones.get() print 'all workers done in %.2f secs' % (time.time() - start) diesel.quickstop() if __name__ == '__main__': diesel.quickstart(main)
else: cls = ConvoySetTimeout if rest: clock = rest[0] values = rest[1:] return cls(ConsensusSet(values, clock)) else: return cls() else: assert result == "SET-OKAY" clock = rest[0] values = rest[1:] return ConsensusSet(values, clock) @call def keep_alive(self): send("KEEPALIVE\r\n") assert until_eol().strip().upper() == "KEEPALIVE-OKAY" cargo = ConvoyNameService([('localhost', 1111), ('localhost', 1112), ('localhost', 1113)]) if __name__ == '__main__': def run(): print cargo.clear("foo") print cargo.set("foo", "bar") print cargo.lookup("foo") quickstop() from diesel import quickstart, quickstop quickstart(cargo, run)
if __name__ == '__main__': def f(): with PostgreSQLClient(database="test", user="******") as client: with client.transact: client.execute("INSERT INTO companies (name) values ($1)" , "JamieCo3") client.execute("INSERT INTO typtest values ($1, $2, $3, $4, $5, $6)" , "string", "string", 14, 155, 23923, True) #for row in client.query("SELECT * FROM companies", buffer=500): # print row.name # print row.id for row in client.query("SELECT * FROM typtest", buffer=500): print row client.execute("UPDATE companies set name = $1 where id < $2", "marky co", 5) print '\n\n~~~done!~~~' def g(): with PostgreSQLClient(database="test", user="******") as client: for x in xrange(500): r = client.query_one( "select * from counters where id = $1 for update", 1) print 'done' from diesel import quickstart quickstart(f, g, g, g, g, g, g, g, g, g, g, g)
LISTENERS = 10 EVENTS = 5 cd = Countdown(LISTENERS * EVENTS) f = Fanout() def listener(x): with f.sub() as q: while True: v = q.get() print '%s <- %s' % (x, v) cd.tick() def teller(): for x in xrange(EVENTS): sleep(2) f.pub(x) def killer(): cd.wait() quickstop() from functools import partial quickstart(killer, teller, *[partial(listener, x) for x in xrange(LISTENERS)])
if __name__ == '__main__': def f(): with PostgreSQLClient(database="test", user="******") as client: with client.transact: client.execute("INSERT INTO companies (name) values ($1)", "JamieCo3") client.execute( "INSERT INTO typtest values ($1, $2, $3, $4, $5, $6)", "string", "string", 14, 155, 23923, True) #for row in client.query("SELECT * FROM companies", buffer=500): # print row.name # print row.id for row in client.query("SELECT * FROM typtest", buffer=500): print row client.execute("UPDATE companies set name = $1 where id < $2", "marky co", 5) print '\n\n~~~done!~~~' def g(): with PostgreSQLClient(database="test", user="******") as client: for x in xrange(500): r = client.query_one( "select * from counters where id = $1 for update", 1) print 'done' from diesel import quickstart quickstart(f, g, g, g, g, g, g, g, g, g, g, g)
def python(): diesel.quickstart(interact_python)
#!/usr/bin/env python # encoding: utf-8 from diesel import quickstart, sleep, quickstop from diesel.util.event import Event def coordinator(): # Pretend to do something ... sleep(3) # Done, fire the event. print "Coordinator done." ev.set() def consumer(): print "Waiting ..." ev.wait() print "The event was triggered!" def late_consumer(): sleep(4) consumer() quickstop() ev = Event() quickstart(coordinator, consumer, consumer, consumer, late_consumer)
if line == "": break parts = line.split() cmd = parts[0] if cmd == "get": key = parts[1] try: val = CACHE[key] send("VALUE %s 0 %d\r\n" % (key, len(val))) send(val + "\r\n") except KeyError: pass send("END\r\n") elif cmd == "set": key = parts[1] #exp = parts[2] #flags = parts[3] length = int(parts[4]) val = receive(length + 2)[:length] CACHE[key] = val send("STORED\r\n") if __name__ == "__main__": quickstart(Service(handle_con, 11211))
'''Test for client to dreadlock network lock service. ''' import uuid from diesel import sleep, quickstart from diesel.protocols.dreadlock import DreadlockService locker = DreadlockService('localhost', 6001) def f(): with locker.hold("foo", 30): id = uuid.uuid4() print "start!", id sleep(2) print "end!", id quickstart(f, f, f, f, f)
assert isinstance(there, Point) # resolve on put p5.put('there', Point(1, 9)) # should resolve b/c safe=True assert len(c.get('testing.pickles', 'there')['content']) == 1 # Doing stuff with different client ids but the same vector clock. c.set_client_id('diff 1') assert b.put('diff', '---') c.set_client_id('diff 2') assert b.put('diff', '+++') assert b.get('diff') == '+++' # Provoking an error try: # Tell Riak to require 10000 nodes to write this before success. c.put('testing', 'error!', 'oh noes!', w=10000) except RiakErrorResp, e: assert e.errcode == 1, e.errcode assert e.errmsg == '{n_val_violation,3}', e.errmsg assert repr(e) == "RiakErrorResp: {n_val_violation,3}", repr(e) except Exception, e: assert 0, "UNEXPECTED EXCEPTION: %r" % e else: assert 0, "DID NOT RAISE" diesel.quickstop() diesel.quickstart(test_client) del Point
import sys from diesel import quickstart, fork_from_thread from diesel.util.queue import Queue from thread import start_new_thread q = Queue() def consume(): while True: v = q.get() print 'DIESEL GOT', v def put(line): q.put(line) def create(): while True: line = sys.stdin.readline() print 'iter!', line fork_from_thread(put, line) start_new_thread(create, ()) quickstart(consume)
from diesel import quickstart, quickstop, sleep from diesel.protocols.zeromq import DieselZMQSocket, zctx, zmq import time def handle_messages(): insock = DieselZMQSocket(zctx.socket(zmq.DEALER), bind="inproc://foo") for x in xrange(500000): msg = insock.recv() assert msg == "yo dawg %s" % x delt = time.time() - t print "500000 messages in %ss (%.1f/s)" % (delt, 500000.0 / delt) quickstop() def send_message(): global t outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), connect="inproc://foo") t = time.time() for x in xrange(500000): outsock.send("yo dawg %s" % x) if x % 1000 == 0: sleep() def tick(): while True: print "Other diesel stuff" sleep(1) quickstart(handle_messages, send_message, tick)
from diesel import quickstart, quickstop from montageclient import MontageClient from user_palm import UserInfo # run with montage/examples/basic_proxy.hs def test_resolution(cl, bucket, key): [data_1, data_2, data_3] = [ UserInfo(uid=i) for i in xrange(1,4) ] first = cl.put(bucket, key, data_1.dumps()) second = cl.put(bucket, key, data_2.dumps()) third = cl.put(bucket, key, data_3.dumps()) resolved = cl.get(bucket, key) assert UserInfo(resolved.data) == data_3, "Failed last write wins resolution" print "\nsuccessfully did last write wins resolution\n" def main(): cl = MontageClient('localhost', 7078) test_resolution(cl, 'u-info', '1') quickstop() quickstart(main)