def __init__ (self, env=None, info=None): self.loginfo_info = info self.python_prompt_env = env or {'prompt': self} loginfo.log ('Python %s on %s' % ( sys.version, sys.platform ), info) Sync_prompt.__init__ (self)
def main ( database=':memory:', host='127.0.0.2', port=3999, precision=1.0, inactive=3.0, local=512, private=1, public=1, log_buffer_size=4096, anetlog_host='', anetlog_port=3998 ): import time started = time.time () from allegra import netstring, loginfo, tcp_server, ansqlite if anetlog_host: from allegra import anetlog anetlog.stdoe ((anetlog_host, anetlog_port)) loginfo.log ( 'ansqlite - Copyright 2007 Laurent Szyster | Copyleft GPL 2.0', 'info' ) conn, Dispatcher = ansqlite.open (database) ansql = Dispatcher.ansql loads = ansqlite.loads for data in netstring.netpipe (( lambda r=sys.stdin.read: r (log_buffer_size) )): ansql (*loads(data)).close () del ansql, loads listen = tcp_server.decorate (tcp_server.Listen ( Dispatcher, (host, port), precision, max=5 ), inactive, local, private, public) async_loop.catch (listen.server_shutdown) def finalize (listen): anetlog.disconnect () conn.close () listen.finalization = finalize del listen loginfo.log ('loaded in %f seconds' % (time.time () - started), 'info')
def pns_peer (resolved, model): if model[3]: assert None == loginfo.log (model[3], 'pns-tcp-open') return True assert None == loginfo.log (model[3], 'pns-tcp-close') return False
def watch (finalized): "inspect the continuation" loginfo.log ('%r - %d: %r' % ( finalized, len (finalized.async_finalized), finalized.async_finalized ))
def pns_peer(resolved, model): if model[3]: assert None == loginfo.log(model[3], 'pns-tcp-open') return True assert None == loginfo.log(model[3], 'pns-tcp-close') return False
def test_file_collector(when): loginfo.log('test File_collector') c = synchronized.File_collector('test.txt') c.collect_incoming_data('this is a test\n') c.collect_incoming_data('this is a test\n') c.collect_incoming_data('this is a test\n') c.found_terminator() async_loop.schedule(when + async_loop.precision, test_file_producer)
def test_unmanaged(): loginfo.log('test_unmanaged') for host in ('999.999.999.999', '127.0.0.2', '127.0.0.1'): dispatcher = async_chat.Dispatcher() if async_client.connect(dispatcher, (host, 80), 3): dispatcher.async_chat_push('GET /prompt.xml HTTP/1.1\r\n' 'Host: %s\r\n' '\r\n' % host)
def async_net_continue (self, data): try: statement, params = loads (data) r, u = self.ansql (statement, params) if u: loginfo.log (data) except sqlite.Error, e: r = e.args[0]
def pns_statement(model): model, error = pns_model.pns_triple(model) if error: loginfo.log(netstring.encode(model), error) return False channel.pns_statement(tuple(model[:3]), model[3]) return True
def log_statement (model): "Log valid statement to STDOUT, errors to STDERR" model, error = pns_model.pns_triple (model) if error: loginfo.log (netstring.encode (model), error) return False loginfo.log (netstring.encode (model)) return True
def log_statement(model): "Log valid statement to STDOUT, errors to STDERR" model, error = pns_model.pns_triple(model) if error: loginfo.log(netstring.encode(model), error) return False loginfo.log(netstring.encode(model)) return True
def test_file_collector (when): loginfo.log ('test File_collector') c = synchronized.File_collector ('test.txt') c.collect_incoming_data ('this is a test\n') c.collect_incoming_data ('this is a test\n') c.collect_incoming_data ('this is a test\n') c.found_terminator () async_loop.schedule ( when + async_loop.precision, test_file_producer )
def test_popen_collector (when): loginfo.log ('test Popen_collector') c = synchronized.popen_collector ( '/python24/python test/subcollect.py' ) c.collect_incoming_data ('this is a test\n\n') c.found_terminator () async_loop.schedule ( when + async_loop.precision, test_popen_reactor )
def test_produce(when): if p.producer_stalled(): async_loop.schedule(when + async_loop.precision, test_produce) else: data = p.more() if data: loginfo.log(data, 'p.more ()') async_loop.schedule(when + async_loop.precision, test_produce) else: async_loop.schedule(when + async_loop.precision, next_test)
def pns_statement (model): model, error = pns_model.pns_triple (model) if error: loginfo.log (netstring.encode (model), error) return False channel.pns_statement ( tuple (model[:3]), model[3] ) return True
def test_unmanaged (): loginfo.log ('test_unmanaged') for host in ('999.999.999.999', '127.0.0.2', '127.0.0.1'): dispatcher = async_chat.Dispatcher () if async_client.connect (dispatcher, (host, 80), 3): dispatcher.async_chat_push ( 'GET /prompt.xml HTTP/1.1\r\n' 'Host: %s\r\n' '\r\n' % host )
def test_popen_reactor (when): loginfo.log ('test Popen_reactor') apipe = synchronized.Popen_reactor ( '/python24/python test/subcollect.py' ) def finalize (apipe): loginfo.log ('%r' % apipe, 'finalized') apipe.finalization = finalize test_producer (when, apipe.producer, (lambda when: None)) apipe.collector.collect_incoming_data ('this is a test\n\n') apipe.collector.found_terminator ()
def test_popen_reactor(when): loginfo.log('test Popen_reactor') apipe = synchronized.Popen_reactor('/python24/python test/subcollect.py') def finalize(apipe): loginfo.log('%r' % apipe, 'finalized') apipe.finalization = finalize test_producer(when, apipe.producer, (lambda when: None)) apipe.collector.collect_incoming_data('this is a test\n\n') apipe.collector.found_terminator()
def io_unmeter(loop): "log meter statistics and remove the metering decoration" meters = io_meters(loop) loginfo.log( 'io-unmetered' ' seconds="%f" run="%d"' ' rps="%f" load="%f"' ' concurrency="%f" activity="%f"' % meters, 'info') del (loop._io_when, loop._io_run, loop._io_load, _io_concurrency, loop._io_activity) loop._io = loop._io_metered return meters
def pns_signal(self, resolved, model): # dump a netstring to STDOUT loginfo.log(netstring.encode(model)) # waits for the echo to finalize and articulate the # next statement, nicely as the peer sends its echo if model[4].startswith('.'): self.pns_articulate() return False # _, ! or ? are simply dropped unless they occur # before ., ... return True
def pns_signal (self, resolved, model): # dump a netstring to STDOUT loginfo.log (netstring.encode (model)) # waits for the echo to finalize and articulate the # next statement, nicely as the peer sends its echo if model[4].startswith ('.'): self.pns_articulate () return False # _, ! or ? are simply dropped unless they occur # before ., ... return True
def collect (): import gc collected = gc.collect () if collected == 0: return assert None == loginfo.log ('%d' % collected, 'collected') for cycle in gc.garbage: try: cylce.finalization = None except: pass assert None == loginfo.log ('%r' % cycle, 'garbage')
def pns_peer(self, resolved, model): ip = model[3] if ip: encoded = '%d:%s,0:,0:,0:,' % (len(ip), ip) loginfo.log(encoded) self.pns_articulate() return True assert None == self.log( 'close' ' sent="%d" received="%d" seconds="%f"' % (self.pns_sent, self.pns_received, time.time() - self.client_when), 'debug') return False
def http_continue (self, http): uri = http.request[1] try: uri = list (HTTP_URI.match (uri).groups ()) except: uri = ['http://', http.collector_headers.get ( 'host', self.presto_path ), uri, '', ''] else: uri[0] = uri[0] or 'http://' uri[1] = uri[1] or http.collector_headers.get ( 'host', self.presto_path ) http.uri = uri about = uri[2].split ('/', 2) about[0] = uri[1] about = http.uri_about = tuple (( unicode (urldecode (n), 'UTF-8') for n in about )) try: # context, subject and maybe predicate # || http://host/folder || http://host/folder/file controller = self.presto_cached[about] except: if len (about) > 2: about = (about[0], about[1]) else: about = (about[0],) try: # context and subject || http://host/folder controller = self.presto_cached[about] except: controller = http_404_close try: stalled = controller (http, about) except: self.loginfo_traceback () http (500) stalled = False try: digest = http.irtd2[4] except: digest = '%x' % id (http) loginfo.log (str (netstring.encode (( ' '.join (http.request), ''.join (uri[:3]), digest ))), '%d' % http.response) return stalled
def http_continue(self, http): uri = http.request[1] try: uri = list(HTTP_URI.match(uri).groups()) except: uri = [ 'http://', http.collector_headers.get('host', self.presto_path), uri, '', '' ] else: uri[0] = uri[0] or 'http://' uri[1] = uri[1] or http.collector_headers.get( 'host', self.presto_path) http.uri = uri about = uri[2].split('/', 2) about[0] = uri[1] about = http.uri_about = tuple( (unicode(urldecode(n), 'UTF-8') for n in about)) try: # context, subject and maybe predicate # || http://host/folder || http://host/folder/file controller = self.presto_cached[about] except: if len(about) > 2: about = (about[0], about[1]) else: about = (about[0], ) try: # context and subject || http://host/folder controller = self.presto_cached[about] except: controller = http_404_close try: stalled = controller(http, about) except: self.loginfo_traceback() http(500) stalled = False try: digest = http.irtd2[4] except: digest = '%x' % id(http) loginfo.log( str( netstring.encode( (' '.join(http.request), ''.join(uri[:3]), digest))), '%d' % http.response) return stalled
def resolved (request): model = list (request.dns_question) if request.dns_resources == None: model.append ('') elif len (request.dns_resources) > 1: model.append (netstring.encode ( request.dns_resources )) elif request.dns_resources: model.append (request.dns_resources[0]) else: model.append ('') model.append (request.dns_peer[0]) loginfo.log (netstring.encode (model))
def main ( root='.', precision=1.0, inactive=3.0, local=512, private=1, public=1, anetlog_host='', anetlog_port=3998, ac_in=16384, ac_out=16384 ): import time started = time.time () from allegra import loginfo, ip_peer, tcp_server, http_server, presto http_server.Dispatcher.ac_in_buffer_size = ac_in http_server.Dispatcher.ac_out_buffer_size = ac_out if anetlog_host: from allegra import anetlog anetlog.stdoe ((anetlog_host, anetlog_port)) def finalize (joined): anetlog.disconnect () loginfo.log ('final', 'info') else: def finalize (joined): loginfo.log ('final', 'info') loginfo.log ( 'Allegra PRESTo' ' - Coyright 2005-2007 Laurent A.V. Szyster' ' | Copyleft GPL 2.0', 'info' ) import glob, os, stat joined = finalization.Finalization () for listen in [ tcp_server.decorate (presto.Listen ( path, addr, precision ), inactive, local, private, public) for path, addr in [ ( p.replace ('\\', '/'), ip_peer.addr (os.path.basename (p), 80) ) for p in glob.glob ('%s/*' % root) if stat.S_ISDIR (os.stat (p)[0]) ] ]: for filename in listen.presto_dir (): listen.presto_load (filename) async_loop.catch (listen.server_shutdown) listen.finalization = joined joined.finalization = finalize loginfo.log ('loaded in %f seconds' % (time.time () - started), 'info')
def test_cycle (): "test a finalizations' cycle collection" one = Watch ('one') two = Watch ('two') three = Watch ('three') four = Watch ('four') finalization.continuation ([one, two, three, four]) four.cycle = three del one, two, three, four async_loop.dispatch () gc.collect () loginfo.log ('garbage: %r' % gc.garbage) for cycled in gc.garbage: cycled.finalization = None async_loop.dispatch ()
def test_cycle(): "test a finalizations' cycle collection" one = Watch('one') two = Watch('two') three = Watch('three') four = Watch('four') finalization.continuation([one, two, three, four]) four.cycle = three del one, two, three, four async_loop.dispatch() gc.collect() loginfo.log('garbage: %r' % gc.garbage) for cycled in gc.garbage: cycled.finalization = None async_loop.dispatch()
def dispatch (): "dispatch I/O, time and finalization events" assert None == loginfo.log ('async_dispatch_start', 'debug') while _dispatched or _scheduled or _finalized or gc.collect () > 0: try: _io (_dispatched, precision, concurrency) _clock () _finalize () except Exit: if not _catched (): break except: loginfo.traceback () assert None == loginfo.log ('async_dispatch_stop', 'debug')
def dispatch(): "dispatch I/O, time and finalization events" assert None == loginfo.log('async_dispatch_start', 'debug') while _dispatched or _scheduled or _finalized or gc.collect() > 0: try: _io(_dispatched, precision, concurrency) _clock() _finalize() except Exit: if not _catched(): break except: loginfo.traceback() assert None == loginfo.log('async_dispatch_stop', 'debug')
def pns_peer (self, resolved, model): ip = model[3] if ip: encoded = '%d:%s,0:,0:,0:,' % (len (ip), ip) loginfo.log (encoded) self.pns_articulate () return True assert None == self.log ( 'close' ' sent="%d" received="%d" seconds="%f"' % ( self.pns_sent, self.pns_received, time.time () - self.client_when ), 'debug' ) return False
def pns_to_xml_continue(self, finalized): "PNS/XML rolledback" assert None == loginfo.log('%r' % finalized.xml_parsed, 'debug') e = finalized.xml_parsed if e and e.xml_valid != None: e.xml_valid(self) self.xml_root = e self.pns_statement = None
def io_meter (loop, when=None): "decorate the loop module's I/O poll function with meters, log info" loop._io_when = when or time.time () loop._io_run = 0 loop._io_load = loop._io_concurrency = loop._io_activity = 0.0 def _io_metering (map, timeout, limit): loop._io_run += 1 dispatched = len (loop._dispatched) concurrent, polled = _io (map, timeout, limit) if concurrent > 0: loop._io_activity += float (active) / concurrent concurrent = float (concurrent) loop._io_load += concurrent / limit loop._io_concurrency += concurrent / dispatched loop._io_metered = loop._io loop._io = _io_metering loginfo.log ('io-metered', 'info')
def xml_to_pns_continue (self, resolved, model): # handle all XML to PNS responses, assert None == loginfo.log (netstring.encode (model)) if self.pns_resolved == resolved: pass # this is the end, at least if the PNS peer does # handle each of the statement in sequence, not # answering the last one before the others. return False
def async_return (self, code): scin = self.collector spout = self.producer scin.async_code = spout.async_code = code self.producer = self.collector = None spout.async_return = None thread_loop.desynchronize (scin) thread_loop.desynchronize (spout) assert None == loginfo.log ('%r' % code, 'debug')
def xml_to_pns_continue(self, resolved, model): # handle all XML to PNS responses, assert None == loginfo.log(netstring.encode(model)) if self.pns_resolved == resolved: pass # this is the end, at least if the PNS peer does # handle each of the statement in sequence, not # answering the last one before the others. return False
def async_return(self, code): scin = self.collector spout = self.producer scin.async_code = spout.async_code = code self.producer = self.collector = None spout.async_return = None thread_loop.desynchronize(scin) thread_loop.desynchronize(spout) assert None == loginfo.log('%r' % code, 'debug')
def test_produce (when): if p.producer_stalled (): async_loop.schedule ( when + async_loop.precision, test_produce ) else: data = p.more () if data: loginfo.log (data, 'p.more ()') async_loop.schedule ( when + async_loop.precision, test_produce ) else: async_loop.schedule ( when + async_loop.precision, next_test )
def io_meter(loop, when=None): "decorate the loop module's I/O poll function with meters, log info" loop._io_when = when or time.time() loop._io_run = 0 loop._io_load = loop._io_concurrency = loop._io_activity = 0.0 def _io_metering(map, timeout, limit): loop._io_run += 1 dispatched = len(loop._dispatched) concurrent, polled = _io(map, timeout, limit) if concurrent > 0: loop._io_activity += float(active) / concurrent concurrent = float(concurrent) loop._io_load += concurrent / limit loop._io_concurrency += concurrent / dispatched loop._io_metered = loop._io loop._io = _io_metering loginfo.log('io-metered', 'info')
def pns_to_xml_continue (self, finalized): "PNS/XML rolledback" assert None == loginfo.log ( '%r' % finalized.xml_parsed, 'debug' ) e = finalized.xml_parsed if e and e.xml_valid != None: e.xml_valid (self) self.xml_root = e self.pns_statement = None
def callback(self, resultset): assert None == loginfo.log('%r' % resultset) try: stmt, param = loads(statements.next()) except StopIteration: self.handle_close() except: self.loginfo_traceback() else: Dispatcher.statements_count += 1 self(self.callback, stmt, param)
def callback (self, resultset): assert None == loginfo.log ('%r' % resultset) try: stmt, param = loads (statements.next ()) except StopIteration: self.handle_close () except: self.loginfo_traceback () else: Dispatcher.statements_count += 1 self (self.callback, stmt, param)
def test_scale (N, M, Finalization=finalization.Finalization, count=False): "test scales on a system" if count: Finalization.count = 0 def finalize (finalization, finalized): Finalization.count += 1 else: finalize = lambda finalization, finalized: None Finalization.__call__ = finalize t = time.clock () c = [] for i in range (M): f = Finalization () c.append (f) for j in range (N): f.finalization = Finalization () f = f.finalization t = time.clock () - t loginfo.log ('%d instanciations: %f seconds' % (N*M, t)) t = time.clock () del c finalization.async_loop.dispatch () t = time.clock () - t try: loginfo.log ('%d finalizations: %f seconds' % ( Finalization.count, t )) except: loginfo.log ('finalization: %f seconds' % t)
def test_scale(N, M, Finalization=finalization.Finalization, count=False): "test scales on a system" if count: Finalization.count = 0 def finalize(finalization, finalized): Finalization.count += 1 else: finalize = lambda finalization, finalized: None Finalization.__call__ = finalize t = time.clock() c = [] for i in range(M): f = Finalization() c.append(f) for j in range(N): f.finalization = Finalization() f = f.finalization t = time.clock() - t loginfo.log('%d instanciations: %f seconds' % (N * M, t)) t = time.clock() del c finalization.async_loop.dispatch() t = time.clock() - t try: loginfo.log('%d finalizations: %f seconds' % (Finalization.count, t)) except: loginfo.log('finalization: %f seconds' % t)
def main(database=':memory:', host='127.0.0.2', port=3999, precision=1.0, inactive=3.0, local=512, private=1, public=1, log_buffer_size=4096, anetlog_host='', anetlog_port=3998): import time started = time.time() from allegra import netstring, loginfo, tcp_server, ansqlite if anetlog_host: from allegra import anetlog anetlog.stdoe((anetlog_host, anetlog_port)) loginfo.log('ansqlite - Copyright 2007 Laurent Szyster | Copyleft GPL 2.0', 'info') conn, Dispatcher = ansqlite.open(database) ansql = Dispatcher.ansql loads = ansqlite.loads for data in netstring.netpipe( (lambda r=sys.stdin.read: r(log_buffer_size))): ansql(*loads(data)).close() del ansql, loads listen = tcp_server.decorate( tcp_server.Listen(Dispatcher, (host, port), precision, max=5), inactive, local, private, public) async_loop.catch(listen.server_shutdown) def finalize(listen): anetlog.disconnect() conn.close() listen.finalization = finalize del listen loginfo.log('loaded in %f seconds' % (time.time() - started), 'info')
def _catched(): "call async_loop.Exit exception catchers" assert None == loginfo.log('async_catch', 'debug') if _catchers: for catcher in tuple(_catchers): if catcher(): _catchers.remove(catcher) return True if __debug__: for dispatcher in _dispatched.values(): loginfo.log('%r' % dispatcher, 'undispatched') for event in _scheduled: loginfo.log('%r' % (event, ), 'unscheduled') for finalized in _finalized: loginfo.log('%r' % (finalized, ), 'unfinalized') return False
def test_pool(pool): loginfo.log('test_pool') for i in range(3): push(pool(), pool.client_name[0])
def test_cache(cache, hosts): loginfo.log('test_cache') for host in hosts: push(cache(async_chat.Dispatcher, (host, 80)), host)
def test_connections(connections, hosts): loginfo.log('test_manager') for host in hosts: push(connections(async_chat.Dispatcher(), (host, 80)), host)
hosts = ('999.999.999.999', '127.0.0.2', '127.0.0.1') def test_managed(): test_connections(connections, hosts) async_loop.dispatch() finalization.collect() test_cache(cache, hosts) async_loop.dispatch() finalization.collect() test_pool(pool) async_loop.dispatch() finalization.collect() loginfo.log('test_managed') connections = async_client.Connections(3, 1) cache = async_client.Cache(3, 1) pool = async_client.Pool(async_chat.Dispatcher, (hosts[-1], 80), 2, 3, 1) test_managed() loginfo.log('test_limited') connections = async_client.Connections(6, 1) cache = async_client.Cache(6, 1) pool = async_client.Pool(async_chat.Dispatcher, (hosts[-1], 80), 2, 6, 1) for client in (connections, cache, pool): async_client.limited(client, 6, (lambda: 1024), (lambda: 1024)) test_managed()
def finalize (joined): loginfo.log ('final', 'info')
def __call__ (self, finalized): loginfo.log ('%r continue %r - %d: %r' % ( self, finalized, len (finalized.async_finalized), finalized.async_finalized ))
import sys, time from allegra import ( loginfo, async_loop, producer, collector, tcp_client, http_client ) loginfo.log ( 'Allegra HTTP/1.1 Client' ' - Copyright 2005 Laurent A.V. Szyster' ' | Copyleft GPL 2.0', 'info' ) try: method, uri, version = sys.argv[1:4] method = method.upper () protocol, url = uri.split ('//') host, path = url.split ('/', 1) addr = host.split (':') if len (addr) < 2: addr.append ('80') host, port = (addr[0], int(addr[1])) except: sys.exit (1) R = C = 1 urlpath = '/' + path if method == 'POST': body = sys.argv[4] if len (sys.argv) > 5: R = int (sys.argv[5]) if len (sys.argv) > 6: C = int (sys.argv[6]) elif method == 'GET' and len (sys.argv) > 4: