Пример #1
0
Файл: zmq.py Проект: 66ru/kombu
 def __init__(self):
     # active channels
     self._channels = set()
     # file descriptor -> channel map
     self._fd_to_chan = {}
     # poll implementation (epoll/kqueue/select)
     self.poller = poll()
Пример #2
0
 def __init__(self):
     # active channels
     self._channels = set()
     # file descriptor -> channel map
     self._fd_to_chan = {}
     # poll implementation (epoll/kqueue/select)
     self.poller = poll()
Пример #3
0
 def __init__(self):
     # active channels
     self._channels = set()
     # file descriptor -> channel map.
     self._fd_to_chan = {}
     # channel -> socket map
     self._chan_to_sock = {}
     # poll implementation (epoll/kqueue/select)
     self._poller = eventio.poll()
Пример #4
0
 def __init__(self):
     # active channels
     self._channels = set()
     # file descriptor -> channel map.
     self._fd_to_chan = {}
     # channel -> socket map
     self._chan_to_sock = {}
     # poll implementation (epoll/kqueue/select)
     self._poller = eventio.poll()
Пример #5
0
def normal_poll():
    pool = ConnectionPool(host='127.0.0.1', port=6379, db=0)
    cli = Redis(connection_pool=pool)
    poller = poll()
    conns = [(key, cli.connection_pool.get_connection('_')) for key in keys]
    register(conns, poller)
    try:
        start_poll(conns, poller, cli)
    except KeyboardInterrupt:
        unregister(conns, poller)
Пример #6
0
 def __init__(self):
     # active channels
     self._channels = set()
     # file descriptor -> channel map.
     self._fd_to_chan = {}
     # channel -> socket map
     self._chan_to_sock = {}
     # poll implementation (epoll/kqueue/select)
     self.poller = poll()
     # one-shot callbacks called after reading from socket.
     self.after_read = set()
Пример #7
0
 def __init__(self):
     # active channels
     self._channels = set()
     # file descriptor -> channel map.
     self._fd_to_chan = {}
     # channel -> socket map
     self._chan_to_sock = {}
     # poll implementation (epoll/kqueue/select)
     self.poller = poll()
     # one-shot callbacks called after reading from socket.
     self.after_read = set()
Пример #8
0
def cluster_poll():
    startup = [
        {
            'host': '127.0.0.1',
            'port': 30001
        },
        {
            'host': '127.0.0.1',
            'port': 30002
        },
        {
            'host': '127.0.0.1',
            'port': 30003
        },
        {
            'host': '127.0.0.1',
            'port': 30004
        },
        {
            'host': '127.0.0.1',
            'port': 30005
        },
        {
            'host': '127.0.0.1',
            'port': 30006
        },
    ]
    pool = ClusterConnectionPool(startup_nodes=startup)
    cli = RedisCluster(connection_pool=pool)
    poller = poll()
    conns = [(key, cli.connection_pool.get_connection_by_key(key))
             for key in keys]
    register(conns, poller)
    try:
        start_poll(conns, poller, cli)
    except KeyboardInterrupt:
        unregister(conns, poller)
Пример #9
0
    def create_loop(self,
                    generator=generator,
                    sleep=sleep,
                    min=min,
                    next=next,
                    Empty=Empty,
                    StopIteration=StopIteration,
                    KeyError=KeyError,
                    READ=READ,
                    WRITE=WRITE,
                    ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        propagate = self.propagate_errors

        while 1:
            todo = self._ready
            self._ready = set()

            for tick_callback in on_tick:
                tick_callback()

            for item in todo:
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            #  print('[[[HUB]]]: %s' % (self.repr_active(),))
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                    #  print('[EVENTS]: %s' % (self.repr_events(events),))
                except ValueError:  # Issue celery/#882
                    return

                for fd, event in events or ():
                    general_error = False
                    if fd in consolidate and \
                            writers.get(fd) is None:
                        to_consolidate.append(fd)
                        continue
                    cb = cbargs = None

                    if event & READ:
                        try:
                            cb, cbargs = readers[fd]
                        except KeyError:
                            self.remove_reader(fd)
                            continue
                    elif event & WRITE:
                        try:
                            cb, cbargs = writers[fd]
                        except KeyError:
                            self.remove_writer(fd)
                            continue
                    elif event & ERR:
                        general_error = True
                    else:
                        logger.info(W_UNKNOWN_EVENT, event, fd)
                        general_error = True

                    if general_error:
                        try:
                            cb, cbargs = (readers.get(fd) or writers.get(fd))
                        except TypeError:
                            pass

                    if cb is None:
                        self.remove(fd)
                        continue

                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if exc.errno != errno.EBADF:
                                raise
                            hub_remove(fd)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fd)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
Пример #10
0
 def _create_poller(self):
     self._poller = poll()
     self._register_fd = self._poller.register
     self._unregister_fd = self._poller.unregister
Пример #11
0
 def start(self):
     self.poller = eventio.poll()
     self.add_events(self.poller)
     self.poller.on_file_change = self.handle_event
     while not self.shutdown_event.is_set():
         self.poller.poll(1)
Пример #12
0
    def create_loop(self,
                    generator=generator,
                    sleep=sleep,
                    min=min,
                    next=next,
                    Empty=Empty,
                    StopIteration=StopIteration,
                    KeyError=KeyError,
                    READ=READ,
                    WRITE=WRITE,
                    ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        todo = self._ready
        propagate = self.propagate_errors

        while 1:
            for tick_callback in on_tick:
                tick_callback()

            while todo:
                item = todo.popleft()
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            #print('[[[HUB]]]: %s' % (self.repr_active(), ))
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                    #print('[EVENTS]: %s' % (self.nepr_events(events or []), ))
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fileno, event in events or ():
                    if fileno in consolidate and \
                            writers.get(fileno) is None:
                        to_consolidate.append(fileno)
                        continue
                    cb = cbargs = None
                    try:
                        if event & READ:
                            cb, cbargs = readers[fileno]
                        elif event & WRITE:
                            cb, cbargs = writers[fileno]
                        elif event & ERR:
                            try:
                                cb, cbargs = (readers.get(fileno)
                                              or writers.get(fileno))
                            except TypeError:
                                pass
                    except (KeyError, Empty):
                        hub_remove(fileno)
                        continue
                    if cb is None:
                        continue
                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                            hub_remove(fileno)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fileno)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
Пример #13
0
Файл: hub.py Проект: Scalr/kombu
    def create_loop(self,
                    generator=generator, sleep=sleep, min=min, next=next,
                    Empty=Empty, StopIteration=StopIteration,
                    KeyError=KeyError, READ=READ, WRITE=WRITE, ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        propagate = self.propagate_errors
        todo = self._ready

        while 1:
            for tick_callback in on_tick:
                tick_callback()

            # To avoid infinite loop where one of the callables adds items
            # to self._ready (via call_soon or otherwise), we take pop only
            # N items from the ready set.
            # N represents the current number of items on the set.
            # That way if a todo adds another one to the ready set,
            # we will break early and allow execution of readers and writers.
            current_todos = len(todo)
            for _ in itertools.repeat(None, current_todos):
                if not todo:
                    break

                item = todo.pop()
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            #  print('[[[HUB]]]: %s' % (self.repr_active(),))
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                    #  print('[EVENTS]: %s' % (self.repr_events(events),))
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fd, event in events or ():
                    general_error = False
                    if fd in consolidate and \
                            writers.get(fd) is None:
                        to_consolidate.append(fd)
                        continue
                    cb = cbargs = None

                    if event & READ:
                        try:
                            cb, cbargs = readers[fd]
                        except KeyError:
                            self.remove_reader(fd)
                            continue
                    elif event & WRITE:
                        try:
                            cb, cbargs = writers[fd]
                        except KeyError:
                            self.remove_writer(fd)
                            continue
                    elif event & ERR:
                        general_error = True
                    else:
                        logger.info(W_UNKNOWN_EVENT, event, fd)
                        general_error = True

                    if general_error:
                        try:
                            cb, cbargs = (readers.get(fd) or
                                          writers.get(fd))
                        except TypeError:
                            pass

                    if cb is None:
                        self.remove(fd)
                        continue

                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if exc.errno != errno.EBADF:
                                raise
                            hub_remove(fd)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fd)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
Пример #14
0
 def start(self):
     """Called by StartStopComponent at worker startup."""
     self.poller = eventio.poll()
Пример #15
0
 def start(self):
     """Called by Hub bootstep at worker startup."""
     self.poller = eventio.poll()
Пример #16
0
    def _loop(
        self,
        propagate=None,
        generator=generator,
        sleep=sleep,
        min=min,
        next=next,
        Empty=Empty,
        StopIteration=StopIteration,
        KeyError=KeyError,
        READ=READ,
        WRITE=WRITE,
        ERR=ERR,
    ):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        remove_ticks = on_tick.difference_update

        while 1:
            outdated_ticks = set()
            for tick_callback in on_tick:
                try:
                    if isinstance(tick_callback, generator):
                        next(tick_callback)
                    else:
                        tick_callback()
                except StopIteration:
                    outdated_ticks.add(tick_callback)
            remove_ticks(outdated_ticks)

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            # print('[[[HUB]]]: %s' % (self.repr_active(), ))
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fileno, event in events or ():
                    if fileno in consolidate and writers.get(fileno) is None:
                        to_consolidate.append(fileno)
                        continue
                    cb = cbargs = None
                    try:
                        if event & READ:
                            cb, cbargs = readers[fileno]
                        elif event & WRITE:
                            cb, cbargs = writers[fileno]
                        elif event & ERR:
                            try:
                                cb, cbargs = readers.get(fileno) or writers.get(fileno)
                            except TypeError:
                                pass
                    except (KeyError, Empty):
                        continue
                    if cb is None:
                        continue
                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except StopIteration:
                            hub_remove(fileno)
                        except Exception:
                            hub_remove(fileno)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
Пример #17
0
 def start(self):
     """Called by Hub bootstep at worker startup."""
     self.poller = eventio.poll()
Пример #18
0
    def create_loop(self,
                    generator=generator,
                    sleep=sleep,
                    min=min,
                    next=next,
                    Empty=Empty,
                    StopIteration=StopIteration,
                    KeyError=KeyError,
                    READ=READ,
                    WRITE=WRITE,
                    ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        propagate = self.propagate_errors
        todo = self._ready

        while 1:
            for tick_callback in on_tick:
                tick_callback()

            # To avoid infinite loop where one of the callables adds items
            # to self._ready (via call_soon or otherwise), we take pop only
            # N items from the ready set.
            # N represents the current number of items on the set.
            # That way if a todo adds another one to the ready set,
            # we will break early and allow execution of readers and writers.
            current_todos = len(todo)
            for _ in itertools.repeat(None, current_todos):
                if not todo:
                    break

                item = todo.pop()
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            #  print('[[[HUB]]]: %s' % (self.repr_active(),))
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                    #  print('[EVENTS]: %s' % (self.repr_events(events),))
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fd, event in events or ():
                    general_error = False
                    if fd in consolidate and \
                            writers.get(fd) is None:
                        to_consolidate.append(fd)
                        continue
                    cb = cbargs = None

                    if event & READ:
                        try:
                            cb, cbargs = readers[fd]
                        except KeyError:
                            self.remove_reader(fd)
                            continue
                    elif event & WRITE:
                        try:
                            cb, cbargs = writers[fd]
                        except KeyError:
                            self.remove_writer(fd)
                            continue
                    elif event & ERR:
                        general_error = True
                    else:
                        logger.info(W_UNKNOWN_EVENT, event, fd)
                        general_error = True

                    if general_error:
                        try:
                            cb, cbargs = (readers.get(fd) or writers.get(fd))
                        except TypeError:
                            pass

                    if cb is None:
                        self.remove(fd)
                        continue

                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if exc.errno != errno.EBADF:
                                raise
                            hub_remove(fd)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fd)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
Пример #19
0
 def __init__(self, timer=None):
     self.fdmap = {}
     self.poller = poll()
     self.timer = Schedule() if timer is None else timer
Пример #20
0
 def start(self):
     self.poller = eventio.poll()
     self.add_events(self.poller)
     self.poller.on_file_change = self.handle_event
     while not self.shutdown_event.is_set():
         self.poller.poll(1)
Пример #21
0
Файл: hub.py Проект: loic/kombu
    def create_loop(self,
                    generator=generator, sleep=sleep, min=min, next=next,
                    Empty=Empty, StopIteration=StopIteration,
                    KeyError=KeyError, READ=READ, WRITE=WRITE, ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        todo = self._ready
        propagate = self.propagate_errors

        while 1:
            for tick_callback in on_tick:
                tick_callback()

            while todo:
                item = todo.popleft()
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            #print('[[[HUB]]]: %s' % (self.repr_active(), ))
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                    #print('[EVENTS]: %s' % (self.nepr_events(events or []), ))
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fileno, event in events or ():
                    if fileno in consolidate and \
                            writers.get(fileno) is None:
                        to_consolidate.append(fileno)
                        continue
                    cb = cbargs = None

                    if event & READ:
                        try:
                            cb, cbargs = readers[fileno]
                        except KeyError:
                            self.remove_reader(fileno)
                            continue
                    elif event & WRITE:
                        try:
                            cb, cbargs = writers[fileno]
                        except KeyError:
                            self.remove_writer(fileno)
                            continue
                    elif event & ERR:
                        try:
                            cb, cbargs = (readers.get(fileno) or
                                          writers.get(fileno))
                        except TypeError:
                            pass

                    if cb is None:
                        continue
                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if exc.errno != errno.EBADF:
                                raise
                            hub_remove(fileno)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fileno)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
Пример #22
0
Файл: hub.py Проект: Scalr/kombu
 def _create_poller(self):
     self.poller = poll()
     self._register_fd = self.poller.register
     self._unregister_fd = self.poller.unregister
Пример #23
0
    def create_loop(self,
                    generator=generator, sleep=sleep, min=min, next=next,
                    Empty=Empty, StopIteration=StopIteration,
                    KeyError=KeyError, READ=READ, WRITE=WRITE, ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        todo = self._ready
        propagate = self.propagate_errors

        while 1:
            for tick_callback in on_tick:
                tick_callback()

            while todo:
                item = todo.popleft()
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fd, event in events or ():
                    general_error = False
                    if fd in consolidate and \
                            writers.get(fd) is None:
                        to_consolidate.append(fd)
                        continue
                    cb = cbargs = None

                    if event & READ:
                        try:
                            cb, cbargs = readers[fd]
                        except KeyError:
                            self.remove_reader(fd)
                            continue
                    elif event & WRITE:
                        try:
                            cb, cbargs = writers[fd]
                        except KeyError:
                            self.remove_writer(fd)
                            continue
                    elif event & ERR:
                        general_error = True
                    else:
                        logger.info(W_UNKNOWN_EVENT, event, fd)
                        general_error = True

                    if general_error:
                        try:
                            cb, cbargs = (readers.get(fd) or
                                          writers.get(fd))
                        except TypeError:
                            pass

                    if cb is None:
                        self.remove(fd)
                        continue

                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                            hub_remove(fd)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fd)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
Пример #24
0
 def start(self):
     """Called by StartStopComponent at worker startup."""
     self.poller = eventio.poll()
Пример #25
0
 def start(self):
     self.poller = poll()