def putter():
log = glog.name("putter")
log.info("putting 100000 things on queue")
for x in xrange(100000):
q.put(x)
sleep()
def _setup_the_logging_system(self):
if not self.log:
if self.selected_log_level is not None:
log_level = self.selected_log_level
else:
log_level = self.default_log_level
log_name = self.name or self.__class__.__name__
self.log = log.name(log_name)
self.log.min_level = log_level
def _setup_the_logging_system(self):
if not self.log:
if self.selected_log_level is not None:
log_level = self.selected_log_level
else:
log_level = self.default_log_level
log_name = self.name or self.__class__.__name__
self.log = log.name(log_name)
self.log.min_level = log_level
def getter():
log = glog.name("getter")
got = 0
while got < 25000:
try:
s = q.get(timeout=3)
sleep()
except QueueTimeout:
log.warning("timeout before getting a value, retrying...")
continue
got += 1
log.info("SUCCESS! got all 25,000")
cd.tick()
from urlparse import urlparse
from flask import Request, Response
from OpenSSL import SSL
utcnow = datetime.utcnow
try:
from http_parser.parser import HttpParser
except ImportError:
from http_parser.pyparser import HttpParser
from diesel import receive, ConnectionClosed, send, log, Client, call, first
SERVER_TAG = 'diesel-http-server'
hlog = log.name("http-error")
HOSTNAME = os.uname()[1] # win32?
def parse_request_line(line):
'''Given a request line, split it into
(method, url, protocol).
'''
items = line.split(' ')
items[0] = items[0].upper()
if len(items) == 2:
return tuple(items) + ('0.9',)
items[1] = urllib.unquote(items[1])
items[2] = items[2].split('/')[-1].strip()
return tuple(items)
'''Example of event firing.
'''
import time
import random
from diesel import (quickstart, quickstop, sleep,
fire, wait, log, loglevels,
set_log_level)
set_log_level(loglevels.DEBUG)
def gunner():
x = 1
while True:
fire('bam', x)
x += 1
sleep()
def sieged():
t = time.time()
while True:
n = wait('bam')
if n % 10000 == 0:
log.info(str(n))
if n == 50000:
delt = time.time() - t
log.debug("50,000 messages in {0:.3f}s {1:.1f}/s)", delt, 50000 / delt)
quickstop()
log = log.name('fire-system')
quickstart(gunner, sieged)
import random
from diesel import (quickstart, quickstop, sleep, fire, wait, log, loglevels,
set_log_level)
set_log_level(loglevels.DEBUG)
def gunner():
x = 1
while True:
fire('bam', x)
x += 1
sleep()
def sieged():
t = time.time()
while True:
n = wait('bam')
if n % 10000 == 0:
log.info(str(n))
if n == 50000:
delt = time.time() - t
log.debug("50,000 messages in {0:.3f}s {1:.1f}/s)", delt,
50000 / delt)
quickstop()
log = log.name('fire-system')
quickstart(gunner, sieged)
def handle_echo(remote_addr):
while True:
message = until('\r\n')
send("you said: %s" % message)
class EchoClient(Client):
@call
def echo(self, message):
send(message + '\r\n')
back = until("\r\n")
return back
log = log.name('echo-system')
def do_echos():
with EchoClient('localhost', 8000,
ssl_ctx=SSL.Context(SSL.TLSv1_METHOD)) as client:
t = time.time()
for x in xrange(5000):
msg = "hello, world #%s!" % x
echo_result = client.echo(msg)
assert echo_result.strip() == "you said: %s" % msg
log.info('5000 loops in {0:.2f}s', time.time() - t)
quickstop()
quickstart(Service(handle_echo, port=8000, ssl_ctx=server_ctx), do_echos)
# vim:ts=4:sw=4:expandtab
'''Simple http client example.
Check out crawler.py for more advanced behaviors involving
many concurrent clients.
'''
from diesel import Application, Loop, log, quickstart, quickstop
from diesel.protocols.http import HttpClient
def req_loop():
with HttpClient('www.jamwt.com', 80) as client:
heads = {'Host' : 'www.jamwt.com'}
log.info(str(client.request('GET', '/Py-TOC/', heads)))
log.info(str(client.request('GET', '/', heads)))
quickstop()
log = log.name('http-client')
quickstart(req_loop)
def __init__(self, name):
self.__dict__['_logger'] = log.name(name)
self.__dict__['name'] = name
import time
from diesel import Service, Client, send, quickstart, quickstop
from diesel import until, call, log
def handle_echo(remote_addr):
while True:
message = until('\r\n')
send("you said: %s" % message)
class EchoClient(Client):
@call
def echo(self, message):
send(message + '\r\n')
back = until("\r\n")
return back
log = log.name('echo-system')
def do_echos():
client = EchoClient('localhost', 8000)
t = time.time()
for x in xrange(5000):
msg = "hello, world #%s!" % x
echo_result = client.echo(msg)
assert echo_result.strip() == "you said: %s" % msg
log.info('5000 loops in {0:.2f}s', time.time() - t)
quickstop()
quickstart(Service(handle_echo, port=8000), do_echos)
def make_logger(self):
self._dlog = log.name('diesel.web+' + self.logger_name)
# vim:ts=4:sw=4:expandtab
'''Simple http client example.
Check out crawler.py for more advanced behaviors involving
many concurrent clients.
'''
from diesel import Application, Loop, log, quickstart, quickstop
from diesel.protocols.http import HttpClient
def req_loop():
for path in ['/Py-TOC', '/']:
with HttpClient('www.jamwt.com', 80) as client:
heads = {'Host' : 'www.jamwt.com'}
log.info(str(client.request('GET', path, heads)))
quickstop()
log = log.name('http-client')
quickstart(req_loop)
def write_file(lpath, body):
bytes.append(len(body))
lpath = (lpath if not lpath.endswith('/') else (lpath + 'index.html')).lstrip('/')
lpath = os.path.join(folder, lpath)
ensure_dirs(lpath)
open(lpath, 'w').write(body)
def follow_loop(lpath):
log.info(" -> %s" % lpath)
with conn_pool.connection as client:
resp = client.request('GET', lpath, heads)
write_file(lpath, resp.data)
bytes = []
count = None
log = glog.name('http-crawler')
def req_loop():
global count
log.info(path)
with conn_pool.connection as client:
resp = client.request('GET', path, heads)
body = resp.data
write_file(path, body)
links = set(get_links(body))
for l in links:
yield l
count = len(links) + 1
def stop():
def __init__(self, name):
self.__dict__['_logger'] = log.name(name)
self.__dict__['name'] = name
(lpath + 'index.html')).lstrip('/')
lpath = os.path.join(folder, lpath)
ensure_dirs(lpath)
open(lpath, 'w').write(body)
def follow_loop(lpath):
log.info(" -> %s" % lpath)
with conn_pool.connection as client:
resp = client.request('GET', lpath, heads)
write_file(lpath, resp.data)
bytes = []
count = None
log = glog.name('http-crawler')
def req_loop():
global count
log.info(path)
with conn_pool.connection as client:
resp = client.request('GET', path, heads)
body = resp.data
write_file(path, body)
links = set(get_links(body))
for l in links:
yield l
count = len(links) + 1
