def putter():
log = glog.sublog("putter", glog.info)
log.info("putting 100000 things on log")
for x in xrange(100000):
q.put(x)
sleep()
def __call__(self):
assert me.id
should_process = self.roles
rlog = log.sublog("convoy-resolver", LOGLVL_DEBUG)
while True:
for r in should_process:
if r in self.roles_wanted:
resp = self.ns.add(r.name(), me.id, r.limit)
ans = None
if type(resp) == ConsensusSet:
self.roles_owned.add(r)
ans = resp
else:
if r in self.roles_owned:
self.roles_owned.remove(r)
if resp.set:
ans = resp.set
else:
ans = self.ns.lookup(r.name())
if ans:
self.role_clocks[r.name()] = ans.clock
for m in self.role_messages[r]:
self.routes[m] = ans.members
if should_process:
self.log_resolution_table(rlog, should_process)
self.table_changes.put(None)
wait_result = self.ns.wait(5, self.role_clocks)
if type(wait_result) == ConvoyWaitDone:
should_process = set([self.role_by_name[wait_result.key]])
else:
should_process = set()
self.ns.alive()
def __call__(self):
assert me.id
should_process = self.roles
rlog = log.sublog("convoy-resolver", LOGLVL_DEBUG)
while True:
for r in should_process:
if r in self.roles_wanted:
resp = self.ns.add(r.name(), me.id, r.limit)
ans = None
if type(resp) == ConsensusSet:
self.roles_owned.add(r)
ans = resp
else:
if r in self.roles_owned:
self.roles_owned.remove(r)
if resp.set:
ans = resp.set
else:
ans = self.ns.lookup(r.name())
if ans:
self.role_clocks[r.name()] = ans.clock
for m in self.role_messages[r]:
self.routes[m] = ans.members
if should_process:
self.log_resolution_table(rlog, should_process)
self.table_changes.put(None)
wait_result = self.ns.wait(5, self.role_clocks)
if type(wait_result) == ConvoyWaitDone:
should_process = set([self.role_by_name[wait_result.key]])
else:
should_process = set()
self.ns.alive()
def getter():
log = glog.sublog("getter", glog.info)
got = 0
while got < 25000:
try:
s = q.get(timeout=3)
sleep()
except QueueTimeout:
log.warn("timeout before getting a value, retrying...")
continue
got += 1
log.info("SUCCESS! got all 25,000")
cd.tick()
def putter():
log = glog.sublog("putter", glog.info)
log.info("putting 50000 things on log")
for x in xrange(50000):
q.put(x)
sleep()
log.info("done, sleeping for 10s...")
sleep(10)
log.info("putting 50000 *more* things on log")
for x in xrange(50000, 100000):
q.put(x)
sleep()
def putter():
log = glog.sublog("putter", glog.info)
log.info("putting 50000 things on log")
for x in xrange(50000):
q.put(x)
sleep()
log.info("done, sleeping for 10s...")
sleep(10)
log.info("putting 50000 *more* things on log")
for x in xrange(50000, 100000):
q.put(x)
sleep()
def getter():
log = glog.sublog("getter", glog.info)
got = 0
while got < 100000:
try:
s = q.get(timeout=3)
except QueueTimeout:
log.warn("timeout before getting a value, retrying...")
continue
assert s == got
got += 1
if got % 10000 == 0:
log.info("up to %s received, sleeping for 0.5s" % got)
sleep(0.5)
log.info("SUCCESS! got all 100,000")
a.halt()
def getter():
log = glog.sublog("getter", glog.info)
got = 0
while got < 100000:
try:
s = q.get(timeout=3)
except QueueTimeout:
log.warn("timeout before getting a value, retrying...")
continue
assert s == got
got += 1
if got % 10000 == 0:
log.info("up to %s received, sleeping for 0.5s" % got)
sleep(0.5)
log.info("SUCCESS! got all 100,000")
a.halt()
client, heads = get_client()
code, heads, body = client.request('GET', lpath, heads)
except ConnectionClosed:
pass
else:
write_file(lpath, body)
files +=1
break
def req_loop():
global links
client, heads = get_client()
log.info(path)
code, heads, body = client.request('GET', path, heads)
write_file(path, body)
links = get_links(body)
for x in xrange(CONCURRENCY):
a.add_loop(Loop(follow_loop))
a = Application()
a.add_loop(Loop(req_loop))
log = log.sublog('http-crawler', log.info)
def stop():
log.info("Fetched %s files in %.3fs with concurrency=%s" % (files, time.time() - t, CONCURRENCY))
a.halt() # stop application
t = time.time()
a.run()
def init_group(hostconfig):
global clog
global group
clog = log.sublog("consensus-server", LOGLVL_DEBUG)
group = HostGroup(hostconfig)
client = EchoClient("localhost", 8013)
while 1:
bar = client.echo("foo %s" % n)
tms = time.asctime()
log.info("[%s] %s: remote service said %r" % (tms, n, bar))
sleep(2)
return _loop
def echo_self_loop(n):
def _loop():
client = EchoClient("localhost", 8013)
while 1:
bar = client.echo_whatup()
tms = time.asctime()
log.info("[%s] %s: (whatup) remote service said %r" % (tms, n, bar))
sleep(3)
return _loop
a = Application()
log = log.sublog("echo-client", log.info)
for x in xrange(5):
a.add_loop(Loop(echo_loop(x)))
for x in xrange(5):
a.add_loop(Loop(echo_self_loop(x)))
a.run()
from diesel import until, call, log
def handle_echo(remote_addr):
while True:
message = until('\r\n')
send("you said: %s" % message)
class EchoClient(Client):
@call
def echo(self, message):
send(message + '\r\n')
back = until("\r\n")
return back
app = Application()
log = log.sublog('echo-system', log.info)
def do_echos():
client = EchoClient('localhost', 8000, ssl_ctx=SSL.Context(SSL.TLSv1_METHOD))
t = time.time()
for x in xrange(5000):
msg = "hello, world #%s!" % x
echo_result = client.echo(msg)
assert echo_result.strip() == "you said: %s" % msg
log.info('5000 loops in %.2fs' % (time.time() - t))
app.halt()
server_ctx = SSL.Context(SSL.TLSv1_METHOD)
server_ctx.use_privatekey_file('snakeoil-key.pem')
server_ctx.use_certificate_file('snakeoil-cert.pem')
app.add_service(Service(handle_echo, port=8000, ssl_ctx=server_ctx))
# vim:ts=4:sw=4:expandtab
'''Simple echo server.
'''
import time
from diesel import Application, Service, until_eol, sleep, log, send, first
def hi_server(addr):
while 1:
ev, val = first(until_eol=True, sleep=3)
if ev == 'sleep':
log.warn('%s timeout!' % time.asctime())
else:
send("you said %s" % val)
app = Application()
log = log.sublog('echo-timeout-server', log.info)
app.add_service(Service(hi_server, 8013))
app.run()
# vim:ts=4:sw=4:expandtab
'''Simple http client example.
Check out crawler.py for more advanced behaviors involving
many concurrent clients.
'''
from diesel import Application, Loop, log
from diesel.protocols.http import HttpClient, HttpHeaders
def req_loop():
with HttpClient('www.jamwt.com', 80) as client:
heads = HttpHeaders()
heads.set('Host', 'www.jamwt.com')
log.info(client.request('GET', '/Py-TOC/', heads))
log.info(client.request('GET', '/', heads))
a.halt()
a = Application()
log = log.sublog('http-client', log.info)
a.add_loop(Loop(req_loop))
a.run()
while 1:
bar = client.echo("foo %s" % n)
tms = time.asctime()
log.info("[%s] %s: remote service said %r" % (tms, n, bar))
sleep(2)
return _loop
def echo_self_loop(n):
def _loop():
client = EchoClient('localhost', 8013)
while 1:
bar = client.echo_whatup()
tms = time.asctime()
log.info("[%s] %s: (whatup) remote service said %r" %
(tms, n, bar))
sleep(3)
return _loop
a = Application()
log = log.sublog('echo-client', log.info)
for x in xrange(5):
a.add_loop(Loop(echo_loop(x)))
for x in xrange(5):
a.add_loop(Loop(echo_self_loop(x)))
a.run()
# vim:ts=4:sw=4:expandtab
"""Simple http client example.
Check out crawler.py for more advanced behaviors involving
many concurrent clients.
"""
from diesel import Application, Loop, log
from diesel.protocols.http import HttpClient, HttpHeaders
def req_loop():
with HttpClient("www.jamwt.com", 80) as client:
heads = HttpHeaders()
heads.set("Host", "www.jamwt.com")
log.info(client.request("GET", "/Py-TOC/", heads))
log.info(client.request("GET", "/", heads))
a.halt()
a = Application()
log = log.sublog("http-client", log.info)
a.add_loop(Loop(req_loop))
a.run()
# vim:ts=4:sw=4:expandtab
'''Example of event firing.
'''
import time
import random
from diesel import Application, Loop, sleep, fire, wait, log
def gunner():
x = 1
while True:
fire('bam', x)
x += 1
sleep()
def sieged():
t = time.time()
while True:
n = wait('bam')
if n % 10000 == 0:
log.info(n)
if n == 50000:
delt = time.time() - t
log.info("50,000 messages in %.3fs (%.1f/s)" % (delt, 50000 / delt))
a.halt()
a = Application()
log = log.sublog('fire-system', log.info)
a.add_loop(Loop(gunner))
a.add_loop(Loop(sieged))
a.run()
# vim:ts=4:sw=4:expandtab
'''Simple echo server.
'''
import time
from diesel import Application, Service, until_eol, sleep, log, send, first
def hi_server(addr):
while 1:
ev, val = first(until_eol=True, sleep=3)
if ev == 'sleep':
log.warn('%s timeout!' % time.asctime())
else:
send("you said %s" % val)
app = Application()
log = log.sublog('echo-timeout-server', log.info)
app.add_service(Service(hi_server, 8013))
app.run()
def init_group(hostconfig):
global clog
global group
clog = log.sublog("consensus-server", LOGLVL_DEBUG)
group = HostGroup(hostconfig)
