def __init__ (self, env=None, info=None):
self.loginfo_info = info
self.python_prompt_env = env or {'prompt': self}
loginfo.log ('Python %s on %s' % (
sys.version, sys.platform
), info)
Sync_prompt.__init__ (self)
def main (
database=':memory:', host='127.0.0.2', port=3999,
precision=1.0, inactive=3.0, local=512, private=1, public=1,
log_buffer_size=4096, anetlog_host='', anetlog_port=3998
):
import time
started = time.time ()
from allegra import netstring, loginfo, tcp_server, ansqlite
if anetlog_host:
from allegra import anetlog
anetlog.stdoe ((anetlog_host, anetlog_port))
loginfo.log (
'ansqlite - Copyright 2007 Laurent Szyster | Copyleft GPL 2.0', 'info'
)
conn, Dispatcher = ansqlite.open (database)
ansql = Dispatcher.ansql
loads = ansqlite.loads
for data in netstring.netpipe ((
lambda r=sys.stdin.read: r (log_buffer_size)
)):
ansql (*loads(data)).close ()
del ansql, loads
listen = tcp_server.decorate (tcp_server.Listen (
Dispatcher, (host, port), precision, max=5
), inactive, local, private, public)
async_loop.catch (listen.server_shutdown)
def finalize (listen):
anetlog.disconnect ()
conn.close ()
listen.finalization = finalize
del listen
loginfo.log ('loaded in %f seconds' % (time.time () - started), 'info')
def pns_peer (resolved, model):
if model[3]:
assert None == loginfo.log (model[3], 'pns-tcp-open')
return True
assert None == loginfo.log (model[3], 'pns-tcp-close')
return False
def watch (finalized):
"inspect the continuation"
loginfo.log ('%r - %d: %r' % (
finalized,
len (finalized.async_finalized),
finalized.async_finalized
))
def pns_peer(resolved, model):
if model[3]:
assert None == loginfo.log(model[3], 'pns-tcp-open')
return True
assert None == loginfo.log(model[3], 'pns-tcp-close')
return False
def test_file_collector(when):
loginfo.log('test File_collector')
c = synchronized.File_collector('test.txt')
c.collect_incoming_data('this is a test\n')
c.collect_incoming_data('this is a test\n')
c.collect_incoming_data('this is a test\n')
c.found_terminator()
async_loop.schedule(when + async_loop.precision, test_file_producer)
def test_unmanaged():
loginfo.log('test_unmanaged')
for host in ('999.999.999.999', '127.0.0.2', '127.0.0.1'):
dispatcher = async_chat.Dispatcher()
if async_client.connect(dispatcher, (host, 80), 3):
dispatcher.async_chat_push('GET /prompt.xml HTTP/1.1\r\n'
'Host: %s\r\n'
'\r\n' % host)
def async_net_continue (self, data):
try:
statement, params = loads (data)
r, u = self.ansql (statement, params)
if u:
loginfo.log (data)
except sqlite.Error, e:
r = e.args[0]
def pns_statement(model):
model, error = pns_model.pns_triple(model)
if error:
loginfo.log(netstring.encode(model), error)
return False
channel.pns_statement(tuple(model[:3]), model[3])
return True
def log_statement (model):
"Log valid statement to STDOUT, errors to STDERR"
model, error = pns_model.pns_triple (model)
if error:
loginfo.log (netstring.encode (model), error)
return False
loginfo.log (netstring.encode (model))
return True
def log_statement(model):
"Log valid statement to STDOUT, errors to STDERR"
model, error = pns_model.pns_triple(model)
if error:
loginfo.log(netstring.encode(model), error)
return False
loginfo.log(netstring.encode(model))
return True
def test_file_collector (when):
loginfo.log ('test File_collector')
c = synchronized.File_collector ('test.txt')
c.collect_incoming_data ('this is a test\n')
c.collect_incoming_data ('this is a test\n')
c.collect_incoming_data ('this is a test\n')
c.found_terminator ()
async_loop.schedule (
when + async_loop.precision, test_file_producer
)
def test_popen_collector (when):
loginfo.log ('test Popen_collector')
c = synchronized.popen_collector (
'/python24/python test/subcollect.py'
)
c.collect_incoming_data ('this is a test\n\n')
c.found_terminator ()
async_loop.schedule (
when + async_loop.precision, test_popen_reactor
)
def test_produce(when):
if p.producer_stalled():
async_loop.schedule(when + async_loop.precision, test_produce)
else:
data = p.more()
if data:
loginfo.log(data, 'p.more ()')
async_loop.schedule(when + async_loop.precision, test_produce)
else:
async_loop.schedule(when + async_loop.precision, next_test)
def pns_statement (model):
model, error = pns_model.pns_triple (model)
if error:
loginfo.log (netstring.encode (model), error)
return False
channel.pns_statement (
tuple (model[:3]), model[3]
)
return True
def test_unmanaged ():
loginfo.log ('test_unmanaged')
for host in ('999.999.999.999', '127.0.0.2', '127.0.0.1'):
dispatcher = async_chat.Dispatcher ()
if async_client.connect (dispatcher, (host, 80), 3):
dispatcher.async_chat_push (
'GET /prompt.xml HTTP/1.1\r\n'
'Host: %s\r\n'
'\r\n' % host
)
def test_popen_reactor (when):
loginfo.log ('test Popen_reactor')
apipe = synchronized.Popen_reactor (
'/python24/python test/subcollect.py'
)
def finalize (apipe):
loginfo.log ('%r' % apipe, 'finalized')
apipe.finalization = finalize
test_producer (when, apipe.producer, (lambda when: None))
apipe.collector.collect_incoming_data ('this is a test\n\n')
apipe.collector.found_terminator ()
def test_popen_reactor(when):
loginfo.log('test Popen_reactor')
apipe = synchronized.Popen_reactor('/python24/python test/subcollect.py')
def finalize(apipe):
loginfo.log('%r' % apipe, 'finalized')
apipe.finalization = finalize
test_producer(when, apipe.producer, (lambda when: None))
apipe.collector.collect_incoming_data('this is a test\n\n')
apipe.collector.found_terminator()
def io_unmeter(loop):
"log meter statistics and remove the metering decoration"
meters = io_meters(loop)
loginfo.log(
'io-unmetered'
' seconds="%f" run="%d"'
' rps="%f" load="%f"'
' concurrency="%f" activity="%f"' % meters, 'info')
del (loop._io_when, loop._io_run, loop._io_load, _io_concurrency,
loop._io_activity)
loop._io = loop._io_metered
return meters
def pns_signal(self, resolved, model):
# dump a netstring to STDOUT
loginfo.log(netstring.encode(model))
# waits for the echo to finalize and articulate the
# next statement, nicely as the peer sends its echo
if model[4].startswith('.'):
self.pns_articulate()
return False
# _, ! or ? are simply dropped unless they occur
# before ., ...
return True
def pns_signal (self, resolved, model):
# dump a netstring to STDOUT
loginfo.log (netstring.encode (model))
# waits for the echo to finalize and articulate the
# next statement, nicely as the peer sends its echo
if model[4].startswith ('.'):
self.pns_articulate ()
return False
# _, ! or ? are simply dropped unless they occur
# before ., ...
return True
def collect ():
import gc
collected = gc.collect ()
if collected == 0:
return
assert None == loginfo.log ('%d' % collected, 'collected')
for cycle in gc.garbage:
try:
cylce.finalization = None
except:
pass
assert None == loginfo.log ('%r' % cycle, 'garbage')
def pns_peer(self, resolved, model):
ip = model[3]
if ip:
encoded = '%d:%s,0:,0:,0:,' % (len(ip), ip)
loginfo.log(encoded)
self.pns_articulate()
return True
assert None == self.log(
'close'
' sent="%d" received="%d" seconds="%f"' %
(self.pns_sent, self.pns_received, time.time() - self.client_when),
'debug')
return False
def http_continue (self, http):
uri = http.request[1]
try:
uri = list (HTTP_URI.match (uri).groups ())
except:
uri = ['http://', http.collector_headers.get (
'host', self.presto_path
), uri, '', '']
else:
uri[0] = uri[0] or 'http://'
uri[1] = uri[1] or http.collector_headers.get (
'host', self.presto_path
)
http.uri = uri
about = uri[2].split ('/', 2)
about[0] = uri[1]
about = http.uri_about = tuple ((
unicode (urldecode (n), 'UTF-8') for n in about
))
try:
# context, subject and maybe predicate
# || http://host/folder || http://host/folder/file
controller = self.presto_cached[about]
except:
if len (about) > 2:
about = (about[0], about[1])
else:
about = (about[0],)
try:
# context and subject || http://host/folder
controller = self.presto_cached[about]
except:
controller = http_404_close
try:
stalled = controller (http, about)
except:
self.loginfo_traceback ()
http (500)
stalled = False
try:
digest = http.irtd2[4]
except:
digest = '%x' % id (http)
loginfo.log (str (netstring.encode ((
' '.join (http.request),
''.join (uri[:3]),
digest
))), '%d' % http.response)
return stalled
def http_continue(self, http):
uri = http.request[1]
try:
uri = list(HTTP_URI.match(uri).groups())
except:
uri = [
'http://',
http.collector_headers.get('host', self.presto_path), uri, '',
''
]
else:
uri[0] = uri[0] or 'http://'
uri[1] = uri[1] or http.collector_headers.get(
'host', self.presto_path)
http.uri = uri
about = uri[2].split('/', 2)
about[0] = uri[1]
about = http.uri_about = tuple(
(unicode(urldecode(n), 'UTF-8') for n in about))
try:
# context, subject and maybe predicate
# || http://host/folder || http://host/folder/file
controller = self.presto_cached[about]
except:
if len(about) > 2:
about = (about[0], about[1])
else:
about = (about[0], )
try:
# context and subject || http://host/folder
controller = self.presto_cached[about]
except:
controller = http_404_close
try:
stalled = controller(http, about)
except:
self.loginfo_traceback()
http(500)
stalled = False
try:
digest = http.irtd2[4]
except:
digest = '%x' % id(http)
loginfo.log(
str(
netstring.encode(
(' '.join(http.request), ''.join(uri[:3]), digest))),
'%d' % http.response)
return stalled
def resolved (request):
model = list (request.dns_question)
if request.dns_resources == None:
model.append ('')
elif len (request.dns_resources) > 1:
model.append (netstring.encode (
request.dns_resources
))
elif request.dns_resources:
model.append (request.dns_resources[0])
else:
model.append ('')
model.append (request.dns_peer[0])
loginfo.log (netstring.encode (model))
def main (
root='.',
precision=1.0, inactive=3.0, local=512, private=1, public=1,
anetlog_host='', anetlog_port=3998,
ac_in=16384, ac_out=16384
):
import time
started = time.time ()
from allegra import loginfo, ip_peer, tcp_server, http_server, presto
http_server.Dispatcher.ac_in_buffer_size = ac_in
http_server.Dispatcher.ac_out_buffer_size = ac_out
if anetlog_host:
from allegra import anetlog
anetlog.stdoe ((anetlog_host, anetlog_port))
def finalize (joined):
anetlog.disconnect ()
loginfo.log ('final', 'info')
else:
def finalize (joined):
loginfo.log ('final', 'info')
loginfo.log (
'Allegra PRESTo'
' - Coyright 2005-2007 Laurent A.V. Szyster'
' | Copyleft GPL 2.0',
'info'
)
import glob, os, stat
joined = finalization.Finalization ()
for listen in [
tcp_server.decorate (presto.Listen (
path, addr, precision
), inactive, local, private, public)
for path, addr in [
(
p.replace ('\\', '/'),
ip_peer.addr (os.path.basename (p), 80)
)
for p in glob.glob ('%s/*' % root)
if stat.S_ISDIR (os.stat (p)[0])
]
]:
for filename in listen.presto_dir ():
listen.presto_load (filename)
async_loop.catch (listen.server_shutdown)
listen.finalization = joined
joined.finalization = finalize
loginfo.log ('loaded in %f seconds' % (time.time () - started), 'info')
def test_cycle ():
"test a finalizations' cycle collection"
one = Watch ('one')
two = Watch ('two')
three = Watch ('three')
four = Watch ('four')
finalization.continuation ([one, two, three, four])
four.cycle = three
del one, two, three, four
async_loop.dispatch ()
gc.collect ()
loginfo.log ('garbage: %r' % gc.garbage)
for cycled in gc.garbage:
cycled.finalization = None
async_loop.dispatch ()
def test_cycle():
"test a finalizations' cycle collection"
one = Watch('one')
two = Watch('two')
three = Watch('three')
four = Watch('four')
finalization.continuation([one, two, three, four])
four.cycle = three
del one, two, three, four
async_loop.dispatch()
gc.collect()
loginfo.log('garbage: %r' % gc.garbage)
for cycled in gc.garbage:
cycled.finalization = None
async_loop.dispatch()
def dispatch ():
"dispatch I/O, time and finalization events"
assert None == loginfo.log ('async_dispatch_start', 'debug')
while _dispatched or _scheduled or _finalized or gc.collect () > 0:
try:
_io (_dispatched, precision, concurrency)
_clock ()
_finalize ()
except Exit:
if not _catched ():
break
except:
loginfo.traceback ()
assert None == loginfo.log ('async_dispatch_stop', 'debug')
def dispatch():
"dispatch I/O, time and finalization events"
assert None == loginfo.log('async_dispatch_start', 'debug')
while _dispatched or _scheduled or _finalized or gc.collect() > 0:
try:
_io(_dispatched, precision, concurrency)
_clock()
_finalize()
except Exit:
if not _catched():
break
except:
loginfo.traceback()
assert None == loginfo.log('async_dispatch_stop', 'debug')
def pns_peer (self, resolved, model):
ip = model[3]
if ip:
encoded = '%d:%s,0:,0:,0:,' % (len (ip), ip)
loginfo.log (encoded)
self.pns_articulate ()
return True
assert None == self.log (
'close'
' sent="%d" received="%d" seconds="%f"' % (
self.pns_sent, self.pns_received,
time.time () - self.client_when
), 'debug'
)
return False
def pns_to_xml_continue(self, finalized):
"PNS/XML rolledback"
assert None == loginfo.log('%r' % finalized.xml_parsed, 'debug')
e = finalized.xml_parsed
if e and e.xml_valid != None:
e.xml_valid(self)
self.xml_root = e
self.pns_statement = None
def io_meter (loop, when=None):
"decorate the loop module's I/O poll function with meters, log info"
loop._io_when = when or time.time ()
loop._io_run = 0
loop._io_load = loop._io_concurrency = loop._io_activity = 0.0
def _io_metering (map, timeout, limit):
loop._io_run += 1
dispatched = len (loop._dispatched)
concurrent, polled = _io (map, timeout, limit)
if concurrent > 0:
loop._io_activity += float (active) / concurrent
concurrent = float (concurrent)
loop._io_load += concurrent / limit
loop._io_concurrency += concurrent / dispatched
loop._io_metered = loop._io
loop._io = _io_metering
loginfo.log ('io-metered', 'info')
def xml_to_pns_continue (self, resolved, model):
# handle all XML to PNS responses,
assert None == loginfo.log (netstring.encode (model))
if self.pns_resolved == resolved:
pass
# this is the end, at least if the PNS peer does
# handle each of the statement in sequence, not
# answering the last one before the others.
return False
def async_return (self, code):
scin = self.collector
spout = self.producer
scin.async_code = spout.async_code = code
self.producer = self.collector = None
spout.async_return = None
thread_loop.desynchronize (scin)
thread_loop.desynchronize (spout)
assert None == loginfo.log ('%r' % code, 'debug')
def xml_to_pns_continue(self, resolved, model):
# handle all XML to PNS responses,
assert None == loginfo.log(netstring.encode(model))
if self.pns_resolved == resolved:
pass
# this is the end, at least if the PNS peer does
# handle each of the statement in sequence, not
# answering the last one before the others.
return False
def async_return(self, code):
scin = self.collector
spout = self.producer
scin.async_code = spout.async_code = code
self.producer = self.collector = None
spout.async_return = None
thread_loop.desynchronize(scin)
thread_loop.desynchronize(spout)
assert None == loginfo.log('%r' % code, 'debug')
def test_produce (when):
if p.producer_stalled ():
async_loop.schedule (
when + async_loop.precision,
test_produce
)
else:
data = p.more ()
if data:
loginfo.log (data, 'p.more ()')
async_loop.schedule (
when + async_loop.precision,
test_produce
)
else:
async_loop.schedule (
when + async_loop.precision,
next_test
)
def io_meter(loop, when=None):
"decorate the loop module's I/O poll function with meters, log info"
loop._io_when = when or time.time()
loop._io_run = 0
loop._io_load = loop._io_concurrency = loop._io_activity = 0.0
def _io_metering(map, timeout, limit):
loop._io_run += 1
dispatched = len(loop._dispatched)
concurrent, polled = _io(map, timeout, limit)
if concurrent > 0:
loop._io_activity += float(active) / concurrent
concurrent = float(concurrent)
loop._io_load += concurrent / limit
loop._io_concurrency += concurrent / dispatched
loop._io_metered = loop._io
loop._io = _io_metering
loginfo.log('io-metered', 'info')
def pns_to_xml_continue (self, finalized):
"PNS/XML rolledback"
assert None == loginfo.log (
'%r' % finalized.xml_parsed, 'debug'
)
e = finalized.xml_parsed
if e and e.xml_valid != None:
e.xml_valid (self)
self.xml_root = e
self.pns_statement = None
def callback(self, resultset):
assert None == loginfo.log('%r' % resultset)
try:
stmt, param = loads(statements.next())
except StopIteration:
self.handle_close()
except:
self.loginfo_traceback()
else:
Dispatcher.statements_count += 1
self(self.callback, stmt, param)
def callback (self, resultset):
assert None == loginfo.log ('%r' % resultset)
try:
stmt, param = loads (statements.next ())
except StopIteration:
self.handle_close ()
except:
self.loginfo_traceback ()
else:
Dispatcher.statements_count += 1
self (self.callback, stmt, param)
def test_scale (N, M, Finalization=finalization.Finalization, count=False):
"test scales on a system"
if count:
Finalization.count = 0
def finalize (finalization, finalized):
Finalization.count += 1
else:
finalize = lambda finalization, finalized: None
Finalization.__call__ = finalize
t = time.clock ()
c = []
for i in range (M):
f = Finalization ()
c.append (f)
for j in range (N):
f.finalization = Finalization ()
f = f.finalization
t = time.clock () - t
loginfo.log ('%d instanciations: %f seconds' % (N*M, t))
t = time.clock ()
del c
finalization.async_loop.dispatch ()
t = time.clock () - t
try:
loginfo.log ('%d finalizations: %f seconds' % (
Finalization.count, t
))
except:
loginfo.log ('finalization: %f seconds' % t)
def test_scale(N, M, Finalization=finalization.Finalization, count=False):
"test scales on a system"
if count:
Finalization.count = 0
def finalize(finalization, finalized):
Finalization.count += 1
else:
finalize = lambda finalization, finalized: None
Finalization.__call__ = finalize
t = time.clock()
c = []
for i in range(M):
f = Finalization()
c.append(f)
for j in range(N):
f.finalization = Finalization()
f = f.finalization
t = time.clock() - t
loginfo.log('%d instanciations: %f seconds' % (N * M, t))
t = time.clock()
del c
finalization.async_loop.dispatch()
t = time.clock() - t
try:
loginfo.log('%d finalizations: %f seconds' % (Finalization.count, t))
except:
loginfo.log('finalization: %f seconds' % t)
def main(database=':memory:',
host='127.0.0.2',
port=3999,
precision=1.0,
inactive=3.0,
local=512,
private=1,
public=1,
log_buffer_size=4096,
anetlog_host='',
anetlog_port=3998):
import time
started = time.time()
from allegra import netstring, loginfo, tcp_server, ansqlite
if anetlog_host:
from allegra import anetlog
anetlog.stdoe((anetlog_host, anetlog_port))
loginfo.log('ansqlite - Copyright 2007 Laurent Szyster | Copyleft GPL 2.0',
'info')
conn, Dispatcher = ansqlite.open(database)
ansql = Dispatcher.ansql
loads = ansqlite.loads
for data in netstring.netpipe(
(lambda r=sys.stdin.read: r(log_buffer_size))):
ansql(*loads(data)).close()
del ansql, loads
listen = tcp_server.decorate(
tcp_server.Listen(Dispatcher, (host, port), precision, max=5),
inactive, local, private, public)
async_loop.catch(listen.server_shutdown)
def finalize(listen):
anetlog.disconnect()
conn.close()
listen.finalization = finalize
del listen
loginfo.log('loaded in %f seconds' % (time.time() - started), 'info')
def _catched():
"call async_loop.Exit exception catchers"
assert None == loginfo.log('async_catch', 'debug')
if _catchers:
for catcher in tuple(_catchers):
if catcher():
_catchers.remove(catcher)
return True
if __debug__:
for dispatcher in _dispatched.values():
loginfo.log('%r' % dispatcher, 'undispatched')
for event in _scheduled:
loginfo.log('%r' % (event, ), 'unscheduled')
for finalized in _finalized:
loginfo.log('%r' % (finalized, ), 'unfinalized')
return False
def test_pool(pool):
loginfo.log('test_pool')
for i in range(3):
push(pool(), pool.client_name[0])
def test_cache(cache, hosts):
loginfo.log('test_cache')
for host in hosts:
push(cache(async_chat.Dispatcher, (host, 80)), host)
def test_connections(connections, hosts):
loginfo.log('test_manager')
for host in hosts:
push(connections(async_chat.Dispatcher(), (host, 80)), host)
hosts = ('999.999.999.999', '127.0.0.2', '127.0.0.1')
def test_managed():
test_connections(connections, hosts)
async_loop.dispatch()
finalization.collect()
test_cache(cache, hosts)
async_loop.dispatch()
finalization.collect()
test_pool(pool)
async_loop.dispatch()
finalization.collect()
loginfo.log('test_managed')
connections = async_client.Connections(3, 1)
cache = async_client.Cache(3, 1)
pool = async_client.Pool(async_chat.Dispatcher, (hosts[-1], 80), 2, 3, 1)
test_managed()
loginfo.log('test_limited')
connections = async_client.Connections(6, 1)
cache = async_client.Cache(6, 1)
pool = async_client.Pool(async_chat.Dispatcher, (hosts[-1], 80), 2, 6, 1)
for client in (connections, cache, pool):
async_client.limited(client, 6, (lambda: 1024), (lambda: 1024))
test_managed()
def finalize (joined):
loginfo.log ('final', 'info')
def __call__ (self, finalized):
loginfo.log ('%r continue %r - %d: %r' % (
self, finalized,
len (finalized.async_finalized),
finalized.async_finalized
))
import sys, time
from allegra import (
loginfo, async_loop, producer, collector, tcp_client,
http_client
)
loginfo.log (
'Allegra HTTP/1.1 Client'
' - Copyright 2005 Laurent A.V. Szyster'
' | Copyleft GPL 2.0', 'info'
)
try:
method, uri, version = sys.argv[1:4]
method = method.upper ()
protocol, url = uri.split ('//')
host, path = url.split ('/', 1)
addr = host.split (':')
if len (addr) < 2:
addr.append ('80')
host, port = (addr[0], int(addr[1]))
except:
sys.exit (1)
R = C = 1
urlpath = '/' + path
if method == 'POST':
body = sys.argv[4]
if len (sys.argv) > 5:
R = int (sys.argv[5])
if len (sys.argv) > 6:
C = int (sys.argv[6])
elif method == 'GET' and len (sys.argv) > 4:
