def __init__(self):
# active channels
self._channels = set()
# file descriptor -> channel map
self._fd_to_chan = {}
# poll implementation (epoll/kqueue/select)
self.poller = poll()
def __init__(self):
# active channels
self._channels = set()
# file descriptor -> channel map
self._fd_to_chan = {}
# poll implementation (epoll/kqueue/select)
self.poller = poll()
def __init__(self):
# active channels
self._channels = set()
# file descriptor -> channel map.
self._fd_to_chan = {}
# channel -> socket map
self._chan_to_sock = {}
# poll implementation (epoll/kqueue/select)
self._poller = eventio.poll()
def __init__(self):
# active channels
self._channels = set()
# file descriptor -> channel map.
self._fd_to_chan = {}
# channel -> socket map
self._chan_to_sock = {}
# poll implementation (epoll/kqueue/select)
self._poller = eventio.poll()
def normal_poll():
pool = ConnectionPool(host='127.0.0.1', port=6379, db=0)
cli = Redis(connection_pool=pool)
poller = poll()
conns = [(key, cli.connection_pool.get_connection('_')) for key in keys]
register(conns, poller)
try:
start_poll(conns, poller, cli)
except KeyboardInterrupt:
unregister(conns, poller)
def __init__(self):
# active channels
self._channels = set()
# file descriptor -> channel map.
self._fd_to_chan = {}
# channel -> socket map
self._chan_to_sock = {}
# poll implementation (epoll/kqueue/select)
self.poller = poll()
# one-shot callbacks called after reading from socket.
self.after_read = set()
def __init__(self):
# active channels
self._channels = set()
# file descriptor -> channel map.
self._fd_to_chan = {}
# channel -> socket map
self._chan_to_sock = {}
# poll implementation (epoll/kqueue/select)
self.poller = poll()
# one-shot callbacks called after reading from socket.
self.after_read = set()
def cluster_poll():
startup = [
{
'host': '127.0.0.1',
'port': 30001
},
{
'host': '127.0.0.1',
'port': 30002
},
{
'host': '127.0.0.1',
'port': 30003
},
{
'host': '127.0.0.1',
'port': 30004
},
{
'host': '127.0.0.1',
'port': 30005
},
{
'host': '127.0.0.1',
'port': 30006
},
]
pool = ClusterConnectionPool(startup_nodes=startup)
cli = RedisCluster(connection_pool=pool)
poller = poll()
conns = [(key, cli.connection_pool.get_connection_by_key(key))
for key in keys]
register(conns, poller)
try:
start_poll(conns, poller, cli)
except KeyboardInterrupt:
unregister(conns, poller)
def create_loop(self,
generator=generator,
sleep=sleep,
min=min,
next=next,
Empty=Empty,
StopIteration=StopIteration,
KeyError=KeyError,
READ=READ,
WRITE=WRITE,
ERR=ERR):
readers, writers = self.readers, self.writers
poll = self.poller.poll
fire_timers = self.fire_timers
hub_remove = self.remove
scheduled = self.timer._queue
consolidate = self.consolidate
consolidate_callback = self.consolidate_callback
on_tick = self.on_tick
propagate = self.propagate_errors
while 1:
todo = self._ready
self._ready = set()
for tick_callback in on_tick:
tick_callback()
for item in todo:
if item:
item()
poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
# print('[[[HUB]]]: %s' % (self.repr_active(),))
if readers or writers:
to_consolidate = []
try:
events = poll(poll_timeout)
# print('[EVENTS]: %s' % (self.repr_events(events),))
except ValueError: # Issue celery/#882
return
for fd, event in events or ():
general_error = False
if fd in consolidate and \
writers.get(fd) is None:
to_consolidate.append(fd)
continue
cb = cbargs = None
if event & READ:
try:
cb, cbargs = readers[fd]
except KeyError:
self.remove_reader(fd)
continue
elif event & WRITE:
try:
cb, cbargs = writers[fd]
except KeyError:
self.remove_writer(fd)
continue
elif event & ERR:
general_error = True
else:
logger.info(W_UNKNOWN_EVENT, event, fd)
general_error = True
if general_error:
try:
cb, cbargs = (readers.get(fd) or writers.get(fd))
except TypeError:
pass
if cb is None:
self.remove(fd)
continue
if isinstance(cb, generator):
try:
next(cb)
except OSError as exc:
if exc.errno != errno.EBADF:
raise
hub_remove(fd)
except StopIteration:
pass
except Exception:
hub_remove(fd)
raise
else:
try:
cb(*cbargs)
except Empty:
pass
if to_consolidate:
consolidate_callback(to_consolidate)
else:
# no sockets yet, startup is probably not done.
sleep(min(poll_timeout, 0.1))
yield
def _create_poller(self):
self._poller = poll()
self._register_fd = self._poller.register
self._unregister_fd = self._poller.unregister
def start(self):
self.poller = eventio.poll()
self.add_events(self.poller)
self.poller.on_file_change = self.handle_event
while not self.shutdown_event.is_set():
self.poller.poll(1)
def create_loop(self,
generator=generator,
sleep=sleep,
min=min,
next=next,
Empty=Empty,
StopIteration=StopIteration,
KeyError=KeyError,
READ=READ,
WRITE=WRITE,
ERR=ERR):
readers, writers = self.readers, self.writers
poll = self.poller.poll
fire_timers = self.fire_timers
hub_remove = self.remove
scheduled = self.timer._queue
consolidate = self.consolidate
consolidate_callback = self.consolidate_callback
on_tick = self.on_tick
todo = self._ready
propagate = self.propagate_errors
while 1:
for tick_callback in on_tick:
tick_callback()
while todo:
item = todo.popleft()
if item:
item()
poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
#print('[[[HUB]]]: %s' % (self.repr_active(), ))
if readers or writers:
to_consolidate = []
try:
events = poll(poll_timeout)
#print('[EVENTS]: %s' % (self.nepr_events(events or []), ))
except ValueError: # Issue 882
raise StopIteration()
for fileno, event in events or ():
if fileno in consolidate and \
writers.get(fileno) is None:
to_consolidate.append(fileno)
continue
cb = cbargs = None
try:
if event & READ:
cb, cbargs = readers[fileno]
elif event & WRITE:
cb, cbargs = writers[fileno]
elif event & ERR:
try:
cb, cbargs = (readers.get(fileno)
or writers.get(fileno))
except TypeError:
pass
except (KeyError, Empty):
hub_remove(fileno)
continue
if cb is None:
continue
if isinstance(cb, generator):
try:
next(cb)
except OSError as exc:
if get_errno(exc) != errno.EBADF:
raise
hub_remove(fileno)
except StopIteration:
pass
except Exception:
hub_remove(fileno)
raise
else:
try:
cb(*cbargs)
except Empty:
pass
if to_consolidate:
consolidate_callback(to_consolidate)
else:
# no sockets yet, startup is probably not done.
sleep(min(poll_timeout, 0.1))
yield
def create_loop(self,
generator=generator, sleep=sleep, min=min, next=next,
Empty=Empty, StopIteration=StopIteration,
KeyError=KeyError, READ=READ, WRITE=WRITE, ERR=ERR):
readers, writers = self.readers, self.writers
poll = self.poller.poll
fire_timers = self.fire_timers
hub_remove = self.remove
scheduled = self.timer._queue
consolidate = self.consolidate
consolidate_callback = self.consolidate_callback
on_tick = self.on_tick
propagate = self.propagate_errors
todo = self._ready
while 1:
for tick_callback in on_tick:
tick_callback()
# To avoid infinite loop where one of the callables adds items
# to self._ready (via call_soon or otherwise), we take pop only
# N items from the ready set.
# N represents the current number of items on the set.
# That way if a todo adds another one to the ready set,
# we will break early and allow execution of readers and writers.
current_todos = len(todo)
for _ in itertools.repeat(None, current_todos):
if not todo:
break
item = todo.pop()
if item:
item()
poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
# print('[[[HUB]]]: %s' % (self.repr_active(),))
if readers or writers:
to_consolidate = []
try:
events = poll(poll_timeout)
# print('[EVENTS]: %s' % (self.repr_events(events),))
except ValueError: # Issue 882
raise StopIteration()
for fd, event in events or ():
general_error = False
if fd in consolidate and \
writers.get(fd) is None:
to_consolidate.append(fd)
continue
cb = cbargs = None
if event & READ:
try:
cb, cbargs = readers[fd]
except KeyError:
self.remove_reader(fd)
continue
elif event & WRITE:
try:
cb, cbargs = writers[fd]
except KeyError:
self.remove_writer(fd)
continue
elif event & ERR:
general_error = True
else:
logger.info(W_UNKNOWN_EVENT, event, fd)
general_error = True
if general_error:
try:
cb, cbargs = (readers.get(fd) or
writers.get(fd))
except TypeError:
pass
if cb is None:
self.remove(fd)
continue
if isinstance(cb, generator):
try:
next(cb)
except OSError as exc:
if exc.errno != errno.EBADF:
raise
hub_remove(fd)
except StopIteration:
pass
except Exception:
hub_remove(fd)
raise
else:
try:
cb(*cbargs)
except Empty:
pass
if to_consolidate:
consolidate_callback(to_consolidate)
else:
# no sockets yet, startup is probably not done.
sleep(min(poll_timeout, 0.1))
yield
def start(self):
"""Called by StartStopComponent at worker startup."""
self.poller = eventio.poll()
def start(self):
"""Called by Hub bootstep at worker startup."""
self.poller = eventio.poll()
def _loop(
self,
propagate=None,
generator=generator,
sleep=sleep,
min=min,
next=next,
Empty=Empty,
StopIteration=StopIteration,
KeyError=KeyError,
READ=READ,
WRITE=WRITE,
ERR=ERR,
):
readers, writers = self.readers, self.writers
poll = self.poller.poll
fire_timers = self.fire_timers
hub_remove = self.remove
scheduled = self.timer._queue
consolidate = self.consolidate
consolidate_callback = self.consolidate_callback
on_tick = self.on_tick
remove_ticks = on_tick.difference_update
while 1:
outdated_ticks = set()
for tick_callback in on_tick:
try:
if isinstance(tick_callback, generator):
next(tick_callback)
else:
tick_callback()
except StopIteration:
outdated_ticks.add(tick_callback)
remove_ticks(outdated_ticks)
poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
# print('[[[HUB]]]: %s' % (self.repr_active(), ))
if readers or writers:
to_consolidate = []
try:
events = poll(poll_timeout)
except ValueError: # Issue 882
raise StopIteration()
for fileno, event in events or ():
if fileno in consolidate and writers.get(fileno) is None:
to_consolidate.append(fileno)
continue
cb = cbargs = None
try:
if event & READ:
cb, cbargs = readers[fileno]
elif event & WRITE:
cb, cbargs = writers[fileno]
elif event & ERR:
try:
cb, cbargs = readers.get(fileno) or writers.get(fileno)
except TypeError:
pass
except (KeyError, Empty):
continue
if cb is None:
continue
if isinstance(cb, generator):
try:
next(cb)
except StopIteration:
hub_remove(fileno)
except Exception:
hub_remove(fileno)
raise
else:
try:
cb(*cbargs)
except Empty:
pass
if to_consolidate:
consolidate_callback(to_consolidate)
else:
# no sockets yet, startup is probably not done.
sleep(min(poll_timeout, 0.1))
yield
def start(self):
"""Called by Hub bootstep at worker startup."""
self.poller = eventio.poll()
def create_loop(self,
generator=generator,
sleep=sleep,
min=min,
next=next,
Empty=Empty,
StopIteration=StopIteration,
KeyError=KeyError,
READ=READ,
WRITE=WRITE,
ERR=ERR):
readers, writers = self.readers, self.writers
poll = self.poller.poll
fire_timers = self.fire_timers
hub_remove = self.remove
scheduled = self.timer._queue
consolidate = self.consolidate
consolidate_callback = self.consolidate_callback
on_tick = self.on_tick
propagate = self.propagate_errors
todo = self._ready
while 1:
for tick_callback in on_tick:
tick_callback()
# To avoid infinite loop where one of the callables adds items
# to self._ready (via call_soon or otherwise), we take pop only
# N items from the ready set.
# N represents the current number of items on the set.
# That way if a todo adds another one to the ready set,
# we will break early and allow execution of readers and writers.
current_todos = len(todo)
for _ in itertools.repeat(None, current_todos):
if not todo:
break
item = todo.pop()
if item:
item()
poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
# print('[[[HUB]]]: %s' % (self.repr_active(),))
if readers or writers:
to_consolidate = []
try:
events = poll(poll_timeout)
# print('[EVENTS]: %s' % (self.repr_events(events),))
except ValueError: # Issue 882
raise StopIteration()
for fd, event in events or ():
general_error = False
if fd in consolidate and \
writers.get(fd) is None:
to_consolidate.append(fd)
continue
cb = cbargs = None
if event & READ:
try:
cb, cbargs = readers[fd]
except KeyError:
self.remove_reader(fd)
continue
elif event & WRITE:
try:
cb, cbargs = writers[fd]
except KeyError:
self.remove_writer(fd)
continue
elif event & ERR:
general_error = True
else:
logger.info(W_UNKNOWN_EVENT, event, fd)
general_error = True
if general_error:
try:
cb, cbargs = (readers.get(fd) or writers.get(fd))
except TypeError:
pass
if cb is None:
self.remove(fd)
continue
if isinstance(cb, generator):
try:
next(cb)
except OSError as exc:
if exc.errno != errno.EBADF:
raise
hub_remove(fd)
except StopIteration:
pass
except Exception:
hub_remove(fd)
raise
else:
try:
cb(*cbargs)
except Empty:
pass
if to_consolidate:
consolidate_callback(to_consolidate)
else:
# no sockets yet, startup is probably not done.
sleep(min(poll_timeout, 0.1))
yield
def __init__(self, timer=None):
self.fdmap = {}
self.poller = poll()
self.timer = Schedule() if timer is None else timer
def start(self):
self.poller = eventio.poll()
self.add_events(self.poller)
self.poller.on_file_change = self.handle_event
while not self.shutdown_event.is_set():
self.poller.poll(1)
def create_loop(self,
generator=generator, sleep=sleep, min=min, next=next,
Empty=Empty, StopIteration=StopIteration,
KeyError=KeyError, READ=READ, WRITE=WRITE, ERR=ERR):
readers, writers = self.readers, self.writers
poll = self.poller.poll
fire_timers = self.fire_timers
hub_remove = self.remove
scheduled = self.timer._queue
consolidate = self.consolidate
consolidate_callback = self.consolidate_callback
on_tick = self.on_tick
todo = self._ready
propagate = self.propagate_errors
while 1:
for tick_callback in on_tick:
tick_callback()
while todo:
item = todo.popleft()
if item:
item()
poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
#print('[[[HUB]]]: %s' % (self.repr_active(), ))
if readers or writers:
to_consolidate = []
try:
events = poll(poll_timeout)
#print('[EVENTS]: %s' % (self.nepr_events(events or []), ))
except ValueError: # Issue 882
raise StopIteration()
for fileno, event in events or ():
if fileno in consolidate and \
writers.get(fileno) is None:
to_consolidate.append(fileno)
continue
cb = cbargs = None
if event & READ:
try:
cb, cbargs = readers[fileno]
except KeyError:
self.remove_reader(fileno)
continue
elif event & WRITE:
try:
cb, cbargs = writers[fileno]
except KeyError:
self.remove_writer(fileno)
continue
elif event & ERR:
try:
cb, cbargs = (readers.get(fileno) or
writers.get(fileno))
except TypeError:
pass
if cb is None:
continue
if isinstance(cb, generator):
try:
next(cb)
except OSError as exc:
if exc.errno != errno.EBADF:
raise
hub_remove(fileno)
except StopIteration:
pass
except Exception:
hub_remove(fileno)
raise
else:
try:
cb(*cbargs)
except Empty:
pass
if to_consolidate:
consolidate_callback(to_consolidate)
else:
# no sockets yet, startup is probably not done.
sleep(min(poll_timeout, 0.1))
yield
def _create_poller(self):
self.poller = poll()
self._register_fd = self.poller.register
self._unregister_fd = self.poller.unregister
def create_loop(self,
generator=generator, sleep=sleep, min=min, next=next,
Empty=Empty, StopIteration=StopIteration,
KeyError=KeyError, READ=READ, WRITE=WRITE, ERR=ERR):
readers, writers = self.readers, self.writers
poll = self.poller.poll
fire_timers = self.fire_timers
hub_remove = self.remove
scheduled = self.timer._queue
consolidate = self.consolidate
consolidate_callback = self.consolidate_callback
on_tick = self.on_tick
todo = self._ready
propagate = self.propagate_errors
while 1:
for tick_callback in on_tick:
tick_callback()
while todo:
item = todo.popleft()
if item:
item()
poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
if readers or writers:
to_consolidate = []
try:
events = poll(poll_timeout)
except ValueError: # Issue 882
raise StopIteration()
for fd, event in events or ():
general_error = False
if fd in consolidate and \
writers.get(fd) is None:
to_consolidate.append(fd)
continue
cb = cbargs = None
if event & READ:
try:
cb, cbargs = readers[fd]
except KeyError:
self.remove_reader(fd)
continue
elif event & WRITE:
try:
cb, cbargs = writers[fd]
except KeyError:
self.remove_writer(fd)
continue
elif event & ERR:
general_error = True
else:
logger.info(W_UNKNOWN_EVENT, event, fd)
general_error = True
if general_error:
try:
cb, cbargs = (readers.get(fd) or
writers.get(fd))
except TypeError:
pass
if cb is None:
self.remove(fd)
continue
if isinstance(cb, generator):
try:
next(cb)
except OSError as exc:
if get_errno(exc) != errno.EBADF:
raise
hub_remove(fd)
except StopIteration:
pass
except Exception:
hub_remove(fd)
raise
else:
try:
cb(*cbargs)
except Empty:
pass
if to_consolidate:
consolidate_callback(to_consolidate)
else:
# no sockets yet, startup is probably not done.
sleep(min(poll_timeout, 0.1))
yield
def start(self):
"""Called by StartStopComponent at worker startup."""
self.poller = eventio.poll()
def start(self):
self.poller = poll()
