def _connection(self, nb, tx, cb):
# Reuse connection
proto, host, port = self.transactor.endpoint(tx)
cid = tx.connection or self._dequeue(nb, "{0}:{1}:{2}".format(proto, host, port), True)
if cid:
if DEBUG:
warn("-- Reusing connection {0} ({1}://{2}:{3})\n".format(cid, proto, host, port))
self._connections[cid] = {'cb': cb, 'nb': nb, 'tx': tx}
if not tx.connection:
tx.kept_alive
self._connected(cid)
return cid
# CONNECT request to proxy required
cid = self._connect_proxy(nb, tx, cb)
if cid:
return cid
# Connect
if DEBUG:
warn("-- Connect ({0}://{1}:{2})\n".format(proto, host, port))
cid = self._connect(nb, True, tx, cid, self._connected)
self._connections[cid] = {'cb': cb, 'nb': nb, 'tx': tx, 'writing': False}
return cid
def start(self, tx, cb=None):
# TODO Pyjo.UserAgent.Server and fork safety
# Non-blocking
if cb:
if DEBUG:
warn("-- Non-blocking request ({0})\n".format(self._url(tx)))
return self._start(True, tx, cb)
# Blocking
else:
if DEBUG:
warn("-- Blocking request ({0})\n".format(self._url(tx)))
class context:
pass
context.tx = tx
def blocking_cb(ua, tx):
ua.ioloop.stop()
context.tx = tx
self._start(False, tx, blocking_cb)
self.ioloop.start()
return context.tx
def _remove(self, taskid):
# Acceptor
if taskid in self._acceptors:
self._acceptors[taskid].unsubscribe('accept').close()
del self._acceptors[taskid]
return self._not_accepting()._maybe_accepting()
# Connections
if taskid in self._connections:
if 'client' in self._connections[taskid]:
self._connections[taskid]['client'].unsubscribe('connect').close()
del self._connections[taskid]
self._maybe_accepting()
if DEBUG:
warn("-- Removed connection {0} ({1} connections)".format(taskid, len(self._connections)))
return
# Timer
reactor = self.reactor
if not reactor:
return
if reactor.remove(taskid):
return
def __del__(self):
if DEBUG:
warn("-- Method {0}.__del__".format(self))
try:
self.close()
except:
pass
def io_cb(reactor, cb, message, write, fd):
if DEBUG:
warn("-- Reactor {0} = {1}".format(message, reactor._ios[fd] if fd in reactor._ios else None))
if DIE:
cb(reactor, write)
else:
try:
cb(reactor, write)
except Exception as e:
reactor.emit('error', e, message)
if self.auto_stop and not reactor._ios and not reactor._timers:
reactor.stop()
def io_cb(reactor, cb, message, write, fd):
if DEBUG:
warn("-- Reactor {0} = {1}".format(
message, reactor._ios[fd] if fd in reactor._ios else None))
if DIE:
cb(reactor, write)
else:
try:
cb(reactor, write)
except Exception as e:
reactor.emit('error', e, message)
if self.auto_stop and not reactor._ios and not reactor._timers:
reactor.stop()
def _timer(self, cb, recurring, after):
tid = None
while True:
tid = md5_sum('t{0}{1}'.format(steady_time(), rand()).encode('ascii'))
if tid not in self._timers:
break
timer = {'cb': cb, 'after': after, 'time': steady_time() + after}
if recurring:
timer['recurring'] = after
self._timers[tid] = timer
if DEBUG:
warn("-- Reactor adding timer[{0}] = {1}".format(tid, self._timers[tid]))
return tid
def recurring(self, cb, after):
"""::
tid = Pyjo.IOLoop.recurring(cb, 3)
tid = loop.recurring(cb, 0)
tid = loop.recurring(cb, 0.25)
Create a new recurring timer, invoking the callback repeatedly after a given
amount of time in seconds. ::
@Pyjo.IOLoop.recurring(5)
def do_something(loop):
...
"""
if DEBUG:
warn("-- Recurring after {0} cb {1}".format(after, cb))
return self._timer(cb, 'recurring', after)
def io(self, cb, handle):
"""::
reactor = reactor.io(cb, handle)
Watch handle for I/O events, invoking the callback whenever handle becomes
readable or writable.
"""
fd = handle.fileno()
if fd in self._ios:
self._ios[fd]['cb'] = cb
if DEBUG:
warn("-- Reactor found io[{0}] = {1}".format(fd, self._ios[fd]))
else:
self._ios[fd] = {'cb': cb}
if DEBUG:
warn("-- Reactor adding io[{0}] = {1}".format(fd, self._ios[fd]))
return self.watch(handle, True, True)
def _stream(self, stream, cid):
# Connect stream with reactor
self._connections[cid] = {'stream': stream}
if DEBUG:
warn("-- New connection {0} ({1} connections)".format(cid, len(self._connections)))
stream.reactor = weakref.proxy(self.reactor)
loop = weakref.proxy(self)
def close_cb(stream):
if dir(loop):
loop._remove(cid)
stream.on(close_cb, 'close')
stream.start()
return cid
def timer(self, cb, after):
"""::
tid = Pyjo.IOLoop.timer(cb, 3)
tid = loop.timer(cb, 0)
tid = loop.timer(cb, 0.25)
Create a new timer, invoking the callback after a given amount of time in
seconds. ::
# Perform operation in 5 seconds
@Pyjo.IOLoop.timer(5)
def timer_cb(loop):
...
"""
if DEBUG:
warn("-- Timer after {0} cb {1}".format(after, cb))
return self._timer(cb, 'timer', after)
def remove(self, remove):
"""::
boolean = reactor.remove(handle)
boolean = reactor.remove(tid)
Remove handle or timer.
"""
if remove is None:
if DEBUG:
warn("-- Reactor remove None")
return
if isinstance(remove, str):
if DEBUG:
if remove in self._timers:
warn("-- Reactor remove timer[{0}] = {1}".format(
remove, self._timers[remove]))
else:
warn("-- Reactor remove timer[{0}] = None".format(remove))
if remove in self._timers:
self._timers[remove]['handler'].cancel()
del self._timers[remove]
else:
if hasattr(remove, 'fileno'):
fd = remove.fileno()
else:
fd = remove
if DEBUG:
if fd in self._ios:
warn("-- Reactor remove fd {0} = {1}".format(
fd, self._ios[fd]))
else:
warn("-- Reactor remove fd {0} = None".format(fd))
if fd in self._ios:
if 'reader' in self._ios[fd]:
self.loop.remove_reader(fd)
if 'writer' in self._ios[fd]:
self.loop.remove_writer(fd)
del self._ios[fd]
def timer_cb(reactor, cb, recurring, after, tid):
if DEBUG:
warn("-- Reactor alarm timer[{0}] = {1}".format(tid, reactor._timers[tid]))
if recurring:
reactor._timers[tid]['handler'] = reactor.loop.call_later(after, timer_cb, reactor, cb, recurring, after, tid)
else:
reactor.remove(tid)
if DIE:
cb(reactor)
else:
try:
cb(reactor)
except Exception as e:
reactor.emit('error', e, 'Timer {0}'.format(tid))
if self.auto_stop and not reactor._ios and not reactor._timers:
reactor.stop()
def remove(self, remove):
"""::
boolean = reactor.remove(handle)
boolean = reactor.remove(tid)
Remove handle or timer.
"""
if remove is None:
if DEBUG:
warn("-- Reactor remove None")
return
if isinstance(remove, str):
if DEBUG:
if remove in self._timers:
warn("-- Reactor remove timer[{0}] = {1}".format(remove, self._timers[remove]))
else:
warn("-- Reactor remove timer[{0}] = None".format(remove))
if remove in self._timers:
del self._timers[remove]
return True
return False
elif remove is not None:
try:
fd = remove.fileno()
if DEBUG:
if fd in self._ios:
warn("-- Reactor remove io[{0}]".format(fd))
if fd in self._inputs:
self._inputs.remove(fd)
if fd in self._outputs:
self._outputs.remove(fd)
if fd in self._ios:
del self._ios[fd]
return True
except socket.error:
if DEBUG:
warn("-- Reactor remove io {0} already closed".format(remove))
return False
def remove(self, remove):
"""::
boolean = reactor.remove(handle)
boolean = reactor.remove(tid)
Remove handle or timer.
"""
if remove is None:
if DEBUG:
warn("-- Reactor remove None")
return
if isinstance(remove, str):
if DEBUG:
if remove in self._timers:
warn("-- Reactor remove timer[{0}] = {1}".format(remove, self._timers[remove]))
else:
warn("-- Reactor remove timer[{0}] = None".format(remove))
if remove in self._timers:
self._timers[remove]['handler'].cancel()
del self._timers[remove]
else:
if hasattr(remove, 'fileno'):
fd = remove.fileno()
else:
fd = remove
if DEBUG:
if fd in self._ios:
warn("-- Reactor remove fd {0} = {1}".format(fd, self._ios[fd]))
else:
warn("-- Reactor remove fd {0} = None".format(fd))
if fd in self._ios:
if 'reader' in self._ios[fd]:
self.loop.remove_reader(fd)
if 'writer' in self._ios[fd]:
self.loop.remove_writer(fd)
del self._ios[fd]
def _timer(self, cb, recurring, after):
tid = None
while True:
tid = md5_sum('t{0}{1}'.format(steady_time(),
rand()).encode('ascii'))
if tid not in self._timers:
break
timer = {'cb': cb, 'after': after, 'recurring': recurring}
if DEBUG:
warn("-- Reactor adding timer[{0}] = {1}".format(tid, timer))
def timer_cb(reactor, cb, recurring, after, tid):
if DEBUG:
warn("-- Reactor alarm timer[{0}] = {1}".format(
tid, reactor._timers[tid]))
if recurring:
reactor._timers[tid]['handler'] = reactor.loop.call_later(
after, timer_cb, reactor, cb, recurring, after, tid)
else:
reactor.remove(tid)
if DIE:
cb(reactor)
else:
try:
cb(reactor)
except Exception as e:
reactor.emit('error', e, 'Timer {0}'.format(tid))
if self.auto_stop and not reactor._ios and not reactor._timers:
reactor.stop()
timer['handler'] = self.loop.call_later(after, timer_cb,
weakref.proxy(self), cb,
recurring, after, tid)
self._timers[tid] = timer
return tid
def _read(self, cid, chunk):
# Corrupted connection
c = self._connections.get(cid)
if not c:
self._connections.pop(cid, None)
return
tx = c.get('tx', None)
if not tx:
return self._remove(cid)
# Process incoming data
if DEBUG:
warn("-- Client <<< Server ({0})\n{1}\n".format(self._url(tx), str(chunk)))
tx.client_read(chunk)
if tx.is_finished:
self._finish(cid, False)
elif tx.is_writing:
self._write(cid)
def timer_cb(reactor, cb, recurring, after, tid):
if DEBUG:
warn("-- Reactor alarm timer[{0}] = {1}".format(
tid, reactor._timers[tid]))
if recurring:
reactor._timers[tid]['handler'] = reactor.loop.call_later(
after, timer_cb, reactor, cb, recurring, after, tid)
else:
reactor.remove(tid)
if DIE:
cb(reactor)
else:
try:
cb(reactor)
except Exception as e:
reactor.emit('error', e, 'Timer {0}'.format(tid))
if self.auto_stop and not reactor._ios and not reactor._timers:
reactor.stop()
def reset(self):
"""::
reactor.reset()
Remove all handles and timers.
"""
loop = self.loop
for fd in self._ios:
io = self._ios[fd]
if io['reader']:
if DEBUG:
warn("-- Reactor reset fd {0} reader".format(fd))
loop.remove_reader(fd)
if io['writer']:
if DEBUG:
warn("-- Reactor reset fd {0} writer".format(fd))
loop.remove_writer(fd)
self._ios = {}
for tid in self._timers:
timer = self._timers[tid]
if timer['handler']:
if DEBUG:
warn("-- Reactor timer[{0}]".format(tid))
timer['handler'].cancel()
self._timers = {}
def reset(self):
"""::
reactor.reset()
Remove all handles and timers.
"""
loop = self.loop
for fd in self._ios:
io = self._ios[fd]
if io['reader']:
if DEBUG:
warn("-- Reactor reset fd {0} reader".format(fd))
loop.remove_reader(fd)
if io['writer']:
if DEBUG:
warn("-- Reactor reset fd {0} writer".format(fd))
loop.remove_writer(fd)
self._ios = {}
for tid in self._timers:
timer = self._timers[tid]
if timer['handler']:
if DEBUG:
warn("-- Reactor timer[{0}]".format(tid))
timer['handler'].cancel()
self._timers = {}
def io(self, cb, handle):
"""::
reactor = reactor.io(cb, handle)
Watch handle for I/O events, invoking the callback whenever handle becomes
readable or writable.
"""
fd = handle.fileno()
if fd in self._ios:
self._ios[fd]['cb'] = cb
if DEBUG:
warn("-- Reactor found io[{0}] = {1}".format(
fd, self._ios[fd]))
else:
self._ios[fd] = {'cb': cb, 'reader': False, 'writer': False}
if DEBUG:
warn("-- Reactor adding io[{0}] = {1}".format(
fd, self._ios[fd]))
return self.watch(handle, True, True)
def _write(self, cid):
# Get and write chunk
if cid not in self._connections:
return
c = self._connections[cid]
if not c:
return
tx = c['tx']
if not tx:
return
if not tx.is_writing:
return
if c['writing']:
return
c['writing'] = True
chunk = tx.client_write()
c['writing'] = False
if DEBUG:
warn("-- Client >>> Server ({0})\n{1}\n".format(self._url(tx), str(chunk)))
stream = self._loop(c['nb']).stream(cid).write(chunk)
if tx.is_finished:
self._finish(cid)
# Continue writing
if not tx.is_writing:
return
def write_cb(ua):
ua._write(cid)
stream.write(b'', cb=write_cb)
def _timer(self, cb, recurring, after):
tid = None
while True:
tid = md5_sum('t{0}{1}'.format(steady_time(), rand()).encode('ascii'))
if tid not in self._timers:
break
timer = {'cb': cb, 'after': after, 'recurring': recurring}
if DEBUG:
warn("-- Reactor adding timer[{0}] = {1}".format(tid, timer))
def timer_cb(reactor, cb, recurring, after, tid):
if DEBUG:
warn("-- Reactor alarm timer[{0}] = {1}".format(tid, reactor._timers[tid]))
if recurring:
reactor._timers[tid]['handler'] = reactor.loop.call_later(after, timer_cb, reactor, cb, recurring, after, tid)
else:
reactor.remove(tid)
if DIE:
cb(reactor)
else:
try:
cb(reactor)
except Exception as e:
reactor.emit('error', e, 'Timer {0}'.format(tid))
if self.auto_stop and not reactor._ios and not reactor._timers:
reactor.stop()
timer['handler'] = self.loop.call_later(after, timer_cb, weakref.proxy(self), cb, recurring, after, tid)
self._timers[tid] = timer
return tid
def emit(self, name, *args, **kwargs):
"""::
e = e.emit('foo')
e = e.emit('foo', 123)
Emit event.
"""
if name in self._events:
s = self._events[name]
if DEBUG:
warn("-- Emit {0} in {1} ({2})".format(name, self, len(s)))
for cb in s:
cb(self, *args, **kwargs)
else:
if DEBUG:
warn("-- Emit {0} in {1} (0)".format(name, self))
if name == 'error':
raise Error(*args, **kwargs)
return self
def watch(self, handle, read, write):
"""::
reactor = reactor.watch(handle, read, write)
Change I/O events to watch handle for with true and false values. Note
that this method requires an active I/O watcher.
"""
fd = handle.fileno()
def io_cb(reactor, cb, message, write, fd):
if DEBUG:
warn("-- Reactor {0} = {1}".format(
message, reactor._ios[fd] if fd in reactor._ios else None))
if DIE:
cb(reactor, write)
else:
try:
cb(reactor, write)
except Exception as e:
reactor.emit('error', e, message)
if self.auto_stop and not reactor._ios and not reactor._timers:
reactor.stop()
if fd not in self._ios:
self._ios[fd] = {'reader': False, 'writer': False}
io = self._ios[fd]
cb = io['cb']
loop = self.loop
if read:
if io['reader']:
loop.remove_reader(fd)
else:
io['reader'] = True
if DEBUG:
warn("-- Reactor add fd {0} reader".format(fd))
loop.add_reader(fd, io_cb, weakref.proxy(self), cb,
"Read fd {0}".format(fd), False, fd)
elif io['reader']:
if DEBUG:
warn("-- Reactor remove fd {0} reader".format(fd))
loop.remove_reader(fd)
io['reader'] = False
if write:
if io['writer']:
loop.remove_writer(fd)
else:
io['writer'] = True
if DEBUG:
warn("-- Reactor add fd {0} writer".format(fd))
loop.add_writer(fd, io_cb, weakref.proxy(self), cb,
"Write fd {0}".format(fd), True, fd)
elif io['writer']:
if DEBUG:
warn("-- Reactor remove fd {0} writer".format(fd))
loop.remove_writer(fd)
io['writer'] = False
return self
def watch(self, handle, read, write):
"""::
reactor = reactor.watch(handle, read, write)
Change I/O events to watch handle for with true and false values. Note
that this method requires an active I/O watcher.
"""
fd = handle.fileno()
def io_cb(reactor, cb, message, write, fd):
if DEBUG:
warn("-- Reactor {0} = {1}".format(message, reactor._ios[fd] if fd in reactor._ios else None))
if DIE:
cb(reactor, write)
else:
try:
cb(reactor, write)
except Exception as e:
reactor.emit('error', e, message)
if self.auto_stop and not reactor._ios and not reactor._timers:
reactor.stop()
if fd not in self._ios:
self._ios[fd] = {'reader': False, 'writer': False}
io = self._ios[fd]
cb = io['cb']
loop = self.loop
if read:
if io['reader']:
loop.remove_reader(fd)
else:
io['reader'] = True
if DEBUG:
warn("-- Reactor add fd {0} reader".format(fd))
loop.add_reader(fd, io_cb, weakref.proxy(self), cb, "Read fd {0}".format(fd), False, fd)
elif io['reader']:
if DEBUG:
warn("-- Reactor remove fd {0} reader".format(fd))
loop.remove_reader(fd)
io['reader'] = False
if write:
if io['writer']:
loop.remove_writer(fd)
else:
io['writer'] = True
if DEBUG:
warn("-- Reactor add fd {0} writer".format(fd))
loop.add_writer(fd, io_cb, weakref.proxy(self), cb, "Write fd {0}".format(fd), True, fd)
elif io['writer']:
if DEBUG:
warn("-- Reactor remove fd {0} writer".format(fd))
loop.remove_writer(fd)
io['writer'] = False
return self
def one_tick(self):
"""::
reactor.one_tick()
Run reactor until an event occurs. Note that this method can recurse back into
the reactor, so you need to be careful. Meant to be overloaded in a subclass.
"""
# Remember state for later
running = self._running
self._running = True
# Wait for one event
last = False
while not last and self._running:
# Stop automatically if there is nothing to watch
if not self._timers and not self._ios:
return self.stop()
# Calculate ideal timeout based on timers
times = [t['time'] for t in self._timers.values()]
if times:
timeout = min(times) - steady_time()
else:
timeout = 0.5
if timeout < 0:
timeout = 0
# I/O
if self._ios and (self._inputs or self._outputs):
try:
readable, writable, exceptional = select.select(self._inputs, self._outputs, self._inputs, timeout)
for fd in list(set([item for sublist in (exceptional, readable, writable) for item in sublist])):
if fd in self._ios:
if fd in readable or fd in exceptional:
io = self._ios[fd]
last = True
self._sandbox(io['cb'], "Read fd {0}".format(fd), False)
if fd in self._ios:
if fd in writable:
io = self._ios[fd]
last = True
self._sandbox(io['cb'], "Write fd {0}".format(fd), True)
except select.error as e:
# Ctrl-c generates EINTR on Python 2.x
if e.args[0] != errno.EINTR:
raise Exception(e)
# Wait for timeout if poll can't be used
elif timeout:
time.sleep(timeout)
# Timers (time should not change in between timers)
now = steady_time()
for tid in list(self._timers):
if tid not in self._timers:
continue
t = self._timers[tid]
if t['time'] > now:
continue
# Recurring timer
if 'recurring' in t:
t['time'] = now + t['recurring']
# Normal timer
else:
self.remove(tid)
last = True
if t['cb']:
if DEBUG:
warn("-- Alarm timer[{0}] = {1}".format(tid, t))
self._sandbox(t['cb'], "Timer {0}".format(tid))
# Restore state if necessary
if self._running:
self._running = running
def __init__(self, **kwargs):
super(Pyjo_IOLoop, self).__init__(**kwargs)
self.max_accepts = kwargs.get('max_accepts', 0)
"""::
max_accepts = loop.max_accepts
loop.max_accepts = 1000
The maximum number of connections this event loop is allowed to accept before
shutting down gracefully without interrupting existing connections, defaults
to ``0``. Setting the value to ``0`` will allow this event loop to accept new
connections indefinitely. Note that up to half of this value can be subtracted
randomly to improve load balancing between multiple server processes.
"""
self.max_connections = kwargs.get('max_connections', 1000)
"""::
max_connections = loop.max_connections
loop.max_connections = 1000
The maximum number of concurrent connections this event loop is allowed to
handle before stopping to accept new incoming connections, defaults to ``1000``.
"""
self.multi_accept = notnone(kwargs.get('multi_accept'), lambda: 50 if self.max_connections > 50 else 1)
"""::
multi = loop.multi_accept
loop.multi_accept = 100
Number of connections to accept at once, defaults to ``50`` or ``1``, depending
on if the value of :attr:`max_connections` is smaller than ``50``.
"""
module = importlib.import_module(Pyjo.Reactor.Base.detect())
self.reactor = notnone(kwargs.get('reactor'), module.new())
"""::
reactor = loop.reactor
loop.reactor = Pyjo.Reactor.new()
Low-level event reactor, usually a :mod:`Pyjo.Reactor.Poll` or
:mod:`Pyjo.Reactor.Select` object with a default subscriber to the event
``error``. ::
# Watch if handle becomes readable or writable
def io_cb(reactor, writable):
if writable:
print('Handle is writable')
else:
print('Handle is readable')
loop.reactor.io(io_cb, handle)
# Change to watching only if handle becomes writable
loop.reactor.watch(handle, read=False, write=True)
# Remove handle again
loop.reactor.remove(handle)
"""
self._accepting_timer = False
self._acceptors = {}
self._accepts = None
self._connections = {}
self._stop_timer = None
if DEBUG:
warn("-- Reactor initialized ({0})".format(self.reactor))
def catch_cb(reactor, e, event):
warn("{0}: {1}\n{2}".format(reactor, event, traceback.format_exc()))
self.reactor.catch(catch_cb)
def catch_cb(reactor, e, event):
warn("{0}: {1}\n{2}".format(reactor, event, traceback.format_exc()))