コード例 #1
0
ファイル: processes.py プロジェクト: jarieb/celery
def _select(self, readers=None, writers=None, err=None, timeout=0):
    readers = set() if readers is None else readers
    writers = set() if writers is None else writers
    err = set() if err is None else err
    try:
        r, w, e = select.select(readers, writers, err, timeout)
        if e:
            seen = set()
            r = r | set(f for f in r + e if f not in seen and not seen.add(f))
        return r, w, 0
    except (select.error, socket.error) as exc:
        if get_errno(exc) == errno.EINTR:
            return [], [], 1
        elif get_errno(exc) in SELECT_BAD_FD:
            for fd in readers | writers | err:
                try:
                    select.select([fd], [], [], 0)
                except (select.error, socket.error) as exc:
                    if get_errno(exc) not in SELECT_BAD_FD:
                        raise
                    readers.discard(fd)
                    writers.discard(fd)
                    err.discard(fd)
            return [], [], 1
        else:
            raise
コード例 #2
0
ファイル: processes.py プロジェクト: dolfly/celery
        def _write_ack(fd, ack, callback=None):
            header, body, body_size = ack._payload
            try:
                try:
                    proc = fileno_to_synq[fd]
                except KeyError:
                    raise StopIteration()
                send_offset = proc.synq._writer.send_offset

                Hw = Bw = 0
                while Hw < 4:
                    try:
                        Hw += send_offset(header, Hw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        yield
                while Bw < body_size:
                    try:
                        Bw += send_offset(body, Bw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        yield
            finally:
                if callback:
                    callback()
                active_writes.discard(fd)
コード例 #3
0
ファイル: asynpool.py プロジェクト: Nextdoor/celery
    def process_flush_queues(self, proc):
        """Flushes all queues, including the outbound buffer, so that
        all tasks that have not been started will be discarded.

        In Celery this is called whenever the transport connection is lost
        (consumer restart).

        """
        resq = proc.outq._reader
        on_state_change = self._result_handler.on_state_change
        fds = set([resq])
        while fds and not resq.closed and self._state != TERMINATE:
            readable, _, again = _select(fds, None, fds, timeout=0.01)
            if readable:
                try:
                    task = resq.recv()
                except (OSError, IOError, EOFError) as exc:
                    if get_errno(exc) == errno.EINTR:
                        continue
                    elif get_errno(exc) == errno.EAGAIN:
                        break
                    else:
                        debug("got %r while flushing process %r", exc, proc, exc_info=1)
                    if get_errno(exc) not in UNAVAIL:
                        debug("got %r while flushing process %r", exc, proc, exc_info=1)
                    break
                else:
                    if task is None:
                        debug("got sentinel while flushing process %r", proc)
                        break
                    else:
                        on_state_change(task)
            else:
                break
コード例 #4
0
ファイル: processes.py プロジェクト: jarieb/celery
        def _write_ack(fd, ack, callback=None):
            # writes ack back to the worker if synack enabled.
            # this operation *MUST* complete, otherwise
            # the worker process will hang waiting for the ack.
            header, body, body_size = ack[2]
            try:
                try:
                    proc = fileno_to_synq[fd]
                except KeyError:
                    # process died, we can safely discard the ack at this
                    # point.
                    raise StopIteration()
                send = proc.send_syn_offset

                Hw = Bw = 0
                while Hw < 4:
                    try:
                        Hw += send(header, Hw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        yield
                while Bw < body_size:
                    try:
                        Bw += send(body, Bw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        yield
            finally:
                if callback:
                    callback()
                # message written, so this fd is now available
                active_writes.discard(fd)
コード例 #5
0
ファイル: processes.py プロジェクト: carlohamalainen/celery
def _select(self, readers=None, writers=None, err=None, timeout=0):
    readers = set() if readers is None else readers
    writers = set() if writers is None else writers
    err = set() if err is None else err
    try:
        r, w, e = select.select(readers, writers, err, timeout)
        if e:
            seen = set()
            r = r | set(f for f in r + e if f not in seen and not seen.add(f))
        return r, w, 0
    except (select.error, socket.error) as exc:
        if get_errno(exc) == errno.EINTR:
            return [], [], 1
        elif get_errno(exc) in SELECT_BAD_FD:
            for fd in readers | writers | err:
                try:
                    select.select([fd], [], [], 0)
                except (select.error, socket.error) as exc:
                    if get_errno(exc) not in SELECT_BAD_FD:
                        raise
                    readers.discard(fd)
                    writers.discard(fd)
                    err.discard(fd)
            return [], [], 1
        else:
            raise
コード例 #6
0
ファイル: processes.py プロジェクト: carlohamalainen/celery
        def _write_ack(fd, ack, callback=None):
            header, body, body_size = ack._payload
            try:
                try:
                    proc = fileno_to_synq[fd]
                except KeyError:
                    raise StopIteration()
                send_offset = proc.synq._writer.send_offset

                Hw = Bw = 0
                while Hw < 4:
                    try:
                        Hw += send_offset(header, Hw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        yield
                while Bw < body_size:
                    try:
                        Bw += send_offset(body, Bw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        yield
            finally:
                if callback:
                    callback()
                active_writes.discard(fd)
コード例 #7
0
ファイル: processes.py プロジェクト: carlohamalainen/celery
        def _write_job(fd, job, callback=None):
            header, body, body_size = job._payload
            try:
                try:
                    proc = fileno_to_inq[fd]
                except KeyError:
                    put_message(job)
                    raise StopIteration()
                # job result keeps track of what process the job is sent to.
                job._write_to = proc
                send_offset = proc.inq._writer.send_offset

                Hw = Bw = 0
                while Hw < 4:
                    try:
                        Hw += send_offset(header, Hw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        yield
                while Bw < body_size:
                    try:
                        Bw += send_offset(body, Bw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        yield
            finally:
                if callback:
                    callback()
                active_writes.discard(fd)
コード例 #8
0
ファイル: processes.py プロジェクト: michaelhenry/celery
        def _write_ack(fd, ack, callback=None):
            # writes ack back to the worker if synack enabled.
            # this operation *MUST* complete, otherwise
            # the worker process will hang waiting for the ack.
            header, body, body_size = ack[2]
            try:
                try:
                    proc = fileno_to_synq[fd]
                except KeyError:
                    # process died, we can safely discard the ack at this
                    # point.
                    raise StopIteration()
                send = proc.send_syn_offset

                Hw = Bw = 0
                while Hw < 4:
                    try:
                        Hw += send(header, Hw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        yield
                while Bw < body_size:
                    try:
                        Bw += send(body, Bw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        yield
            finally:
                if callback:
                    callback()
                # message written, so this fd is now available
                active_writes.discard(fd)
コード例 #9
0
        def _write_to(fd, job, callback=None):
            header, body, body_size = job._payload
            try:
                try:
                    proc = fileno_to_inq[fd]
                except KeyError:
                    put_message(job)
                    raise StopIteration()
                send_offset = proc.inq._writer.send_offset
                # job result keeps track of what process the job is sent to.
                job._write_to = proc

                Hw = Bw = 0
                while Hw < 4:
                    try:
                        Hw += send_offset(header, Hw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        yield
                while Bw < body_size:
                    try:
                        Bw += send_offset(body, Bw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        yield
            finally:
                if callback:
                    callback()
                active_writes.discard(fd)
コード例 #10
0
ファイル: asynpool.py プロジェクト: Nextdoor/celery
    def _recv_message(
        self,
        add_reader,
        fd,
        callback,
        __read__=__read__,
        readcanbuf=readcanbuf,
        BytesIO=BytesIO,
        unpack_from=unpack_from,
        load=_pickle.load,
    ):
        Hr = Br = 0
        if readcanbuf:
            buf = bytearray(4)
            bufv = memoryview(buf)
        else:
            buf = bufv = BytesIO()
        # header
        assert not isblocking(fd)

        while Hr < 4:
            try:
                n = __read__(fd, bufv[Hr:] if readcanbuf else bufv, 4 - Hr)
            except OSError as exc:
                if get_errno(exc) not in UNAVAIL:
                    raise
                yield
            else:
                if n == 0:
                    raise (OSError("End of file during message") if Hr else EOFError())
                Hr += n

        body_size, = unpack_from(">i", bufv)
        if readcanbuf:
            buf = bytearray(body_size)
            bufv = memoryview(buf)
        else:
            buf = bufv = BytesIO()

        while Br < body_size:
            try:
                n = __read__(fd, bufv[Br:] if readcanbuf else bufv, body_size - Br)
            except OSError as exc:
                if get_errno(exc) not in UNAVAIL:
                    raise
                yield
            else:
                if n == 0:
                    raise (OSError("End of file during message") if Br else EOFError())
                Br += n
        add_reader(fd, self.handle_event, fd)
        if readcanbuf:
            message = load(BytesIO(bufv))
        else:
            bufv.seek(0)
            message = load(bufv)
        if message:
            callback(message)
コード例 #11
0
ファイル: asynpool.py プロジェクト: stavxyz/celery
    def _recv_message(self, add_reader, fd, callback,
                      __read__=__read__, readcanbuf=readcanbuf,
                      BytesIO=BytesIO, unpack_from=unpack_from,
                      load=_pickle.load):
        Hr = Br = 0
        if readcanbuf:
            buf = bytearray(4)
            bufv = memoryview(buf)
        else:
            buf = bufv = BytesIO()
        # header
        assert not isblocking(fd)

        while Hr < 4:
            try:
                n = __read__(
                    fd, bufv[Hr:] if readcanbuf else bufv, 4 - Hr,
                )
            except OSError as exc:
                if get_errno(exc) not in UNAVAIL:
                    raise
                yield
            else:
                if n == 0:
                    raise (OSError('End of file during message') if Hr
                           else EOFError())
                Hr += n

        body_size, = unpack_from('>i', bufv)
        if readcanbuf:
            buf = bytearray(body_size)
            bufv = memoryview(buf)
        else:
            buf = bufv = BytesIO()

        while Br < body_size:
            try:
                n = __read__(
                    fd, bufv[Br:] if readcanbuf else bufv, body_size - Br,
                )
            except OSError as exc:
                if get_errno(exc) not in UNAVAIL:
                    raise
                yield
            else:
                if n == 0:
                    raise (OSError('End of file during message') if Br
                           else EOFError())
                Br += n
        add_reader(fd, self.handle_event, fd)
        if readcanbuf:
            message = load(BytesIO(bufv))
        else:
            bufv.seek(0)
            message = load(bufv)
        if message:
            callback(message)
コード例 #12
0
ファイル: asynpool.py プロジェクト: sabw8217/celery_test
        def _write_job(proc, fd, job):
            print 'writing job to %s' % fd
            # writes job to the worker process.
            # Operation must complete if more than one byte of data
            # was written.  If the broker connection is lost
            # and no data was written the operation shall be cancelled.
            header, body, body_size = job._payload
            errors = 0
            try:
                # job result keeps track of what process the job is sent to.
                job._write_to = proc
                send = proc.send_job_offset

                Hw = Bw = 0
                # write header
                while Hw < 4:
                    try:
                        Hw += send(header, Hw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        errors += 1
                        if errors > 100:
                            on_not_recovering(proc, fd, job)
                            raise StopIteration()
                        yield
                    else:
                        errors = 0

                # write body
                while Bw < body_size:
                    try:
                        Bw += send(body, Bw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        errors += 1
                        if errors > 100:
                            on_not_recovering(proc, fd, job)
                            raise StopIteration()
                        yield
                    else:
                        errors = 0
            finally:
                hub_remove(fd)
                write_stats[proc.index] += 1
                # message written, so this fd is now available
                active_writes.discard(fd)
                write_generator_done(job._writer())  # is a weakref
コード例 #13
0
        def _write_job(proc, fd, job):
            # writes job to the worker process.
            # Operation must complete if more than one byte of data
            # was written.  If the broker connection is lost
            # and no data was written the operation shall be cancelled.
            header, body, body_size = job._payload
            errors = 0
            try:
                # job result keeps track of what process the job is sent to.
                job._write_to = proc
                send = proc.send_job_offset

                Hw = Bw = 0
                # write header
                while Hw < 4:
                    try:
                        Hw += send(header, Hw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        errors += 1
                        if errors > 100:
                            on_not_recovering(proc, fd, job)
                            raise StopIteration()
                        yield
                    else:
                        errors = 0

                # write body
                while Bw < body_size:
                    try:
                        Bw += send(body, Bw)
                    except Exception as exc:
                        if get_errno(exc) not in UNAVAIL:
                            raise
                        # suspend until more data
                        errors += 1
                        if errors > 100:
                            on_not_recovering(proc, fd, job)
                            raise StopIteration()
                        yield
                    else:
                        errors = 0
            finally:
                hub_remove(fd)
                write_stats[proc.index] += 1
                # message written, so this fd is now available
                active_writes.discard(fd)
                write_generator_done(job._writer())  # is a weakref
コード例 #14
0
ファイル: asynpool.py プロジェクト: zhangqian2013/celery
    def _recv_message(self,
                      add_reader,
                      fd,
                      callback,
                      read=os.read,
                      unpack=struct.unpack,
                      loads=_pickle.loads,
                      BytesIO=BytesIO):
        buf = BytesIO()
        # header
        remaining = 4
        bsize = None
        assert not isblocking(fd)
        while remaining > 0:
            try:
                bsize = read(fd, remaining)
            except OSError as exc:
                if get_errno(exc) not in UNAVAIL:
                    raise
                yield
            else:
                n = len(bsize)
                if n == 0:
                    if remaining == 4:
                        raise EOFError()
                    else:
                        raise OSError("Got end of file during message")
                remaining -= n

        remaining, = size, = unpack('>i', bsize)
        while remaining > 0:
            try:
                chunk = read(fd, remaining)
            except OSError as exc:
                if get_errno(exc) not in UNAVAIL:
                    raise
                yield
            n = len(chunk)
            if n == 0:
                if remaining == size:
                    raise EOFError()
                else:
                    raise IOError('Got end of file during message')
            buf.write(chunk)
            remaining -= n
        add_reader(fd, self.handle_event, fd)
        message = loads(buf.getvalue())
        if message:
            callback(message)
コード例 #15
0
ファイル: hub.py プロジェクト: 571451370/devstack_mitaka
 def fire_timers(self,
                 min_delay=1,
                 max_delay=10,
                 max_timers=10,
                 propagate=()):
     timer = self.timer
     delay = None
     if timer and timer._queue:
         for i in range(max_timers):
             delay, entry = next(self.scheduler)
             if entry is None:
                 break
             try:
                 entry()
             except propagate:
                 raise
             except (MemoryError, AssertionError):
                 raise
             except OSError as exc:
                 if get_errno(exc) == errno.ENOMEM:
                     raise
                 logger.error('Error in timer: %r', exc, exc_info=1)
             except Exception as exc:
                 logger.error('Error in timer: %r', exc, exc_info=1)
     return min(delay or min_delay, max_delay)
コード例 #16
0
ファイル: asynpool.py プロジェクト: stavxyz/celery
    def process_flush_queues(self, proc):
        """Flushes all queues, including the outbound buffer, so that
        all tasks that have not been started will be discarded.

        In Celery this is called whenever the transport connection is lost
        (consumer restart).

        """
        resq = proc.outq._reader
        on_state_change = self._result_handler.on_state_change
        fds = set([resq])
        while fds and not resq.closed and self._state != TERMINATE:
            readable, _, again = _select(fds, None, fds, timeout=0.01)
            if readable:
                try:
                    task = resq.recv()
                except (OSError, IOError, EOFError) as exc:
                    if get_errno(exc) not in UNAVAIL:
                        debug('got %r while flushing process %r',
                              exc, proc, exc_info=1)
                    break
                else:
                    if task is None:
                        debug('got sentinel while flushing process %r', proc)
                        break
                    else:
                        on_state_change(task)
            else:
                break
コード例 #17
0
ファイル: processes.py プロジェクト: jarieb/celery
 def _stop_task_handler(task_handler):
     for proc in task_handler.pool:
         proc.inq._writer.setblocking(1)
         try:
             proc.inq.put(None)
         except OSError as exc:
             if get_errno(exc) != errno.EBADF:
                 raise
コード例 #18
0
ファイル: processes.py プロジェクト: carlohamalainen/celery
 def _stop_task_handler(task_handler):
     for proc in task_handler.pool:
         proc.inq._writer.setblocking(1)
         try:
             proc.inq.put(None)
         except OSError as exc:
             if get_errno(exc) != errno.EBADF:
                 raise
コード例 #19
0
ファイル: asynpool.py プロジェクト: andrewkittredge/celery
 def _flush_writer(self, writer):
     try:
         list(writer)
     except (OSError, IOError) as exc:
         if get_errno(exc) != errno.EBADF:
             raise
     finally:
         self._active_writers.discard(writer)
コード例 #20
0
ファイル: asynpool.py プロジェクト: rhayesbite/celery
 def _flush_writer(self, writer):
     try:
         list(writer)
     except (OSError, IOError) as exc:
         if get_errno(exc) != errno.EBADF:
             raise
     finally:
         self._active_writers.discard(writer)
コード例 #21
0
ファイル: asynpool.py プロジェクト: dz0ny/celery
    def _recv_message(self, add_reader, fd, callback,
                      read=os.read, unpack=struct.unpack,
                      loads=_pickle.loads, BytesIO=BytesIO):
        buf = BytesIO()
        # header
        remaining = 4
        bsize = None
        assert not isblocking(fd)
        while remaining > 0:
            try:
                bsize = read(fd, remaining)
            except OSError as exc:
                if get_errno(exc) not in UNAVAIL:
                    raise
                yield
            else:
                n = len(bsize)
                if n == 0:
                    if remaining == 4:
                        raise EOFError()
                    else:
                        raise OSError("Got end of file during message")
                remaining -= n

        remaining, = size, = unpack('>i', bsize)
        while remaining > 0:
            try:
                chunk = read(fd, remaining)
            except OSError as exc:
                if get_errno(exc) not in UNAVAIL:
                    raise
                yield
            else:
                n = len(chunk)
                if n == 0:
                    if remaining == size:
                        raise EOFError()
                    else:
                        raise IOError('Got end of file during message')
                buf.write(chunk)
                remaining -= n
        add_reader(fd, self.handle_event, fd)
        message = loads(buf.getvalue())
        if message:
            callback(message)
コード例 #22
0
 def _stop_task_handler(task_handler):
     """Called at shutdown to tell processes that we are shutting down."""
     for proc in task_handler.pool:
         proc.inq._writer.setblocking(1)
         try:
             proc.inq.put(None)
         except OSError as exc:
             if get_errno(exc) != errno.EBADF:
                 raise
コード例 #23
0
ファイル: asynpool.py プロジェクト: subssn21/celery
 def _stop_task_handler(task_handler):
     """Called at shutdown to tell processes that we are shutting down."""
     for proc in task_handler.pool:
         proc.inq._writer.setblocking(1)
         try:
             proc.inq.put(None)
         except OSError as exc:
             if get_errno(exc) != errno.EBADF:
                 raise
コード例 #24
0
ファイル: asynpool.py プロジェクト: subssn21/celery
        def schedule_writes(ready_fds, shuffle=random.shuffle):
            # Schedule write operation to ready file descriptor.
            # The file descriptor is writeable, but that does not
            # mean the process is currently reading from the socket.
            # The socket is buffered so writeable simply means that
            # the buffer can accept at least 1 byte of data.
            shuffle(ready_fds)
            for ready_fd in ready_fds:
                if ready_fd in active_writes:
                    # already writing to this fd
                    continue
                if is_fair_strategy and ready_fd in busy_workers:
                    # worker is already busy with another task
                    continue
                if ready_fd not in all_inqueues:
                    hub_remove(ready_fd)
                    continue
                try:
                    job = pop_message()
                except IndexError:
                    # no more messages, remove all inactive fds from the hub.
                    # this is important since the fds are always writeable
                    # as long as there's 1 byte left in the buffer, and so
                    # this may create a spinloop where the event loop
                    # always wakes up.
                    for inqfd in diff(active_writes):
                        hub_remove(inqfd)
                    break

                else:
                    if not job._accepted:  # job not accepted by another worker
                        try:
                            # keep track of what process the write operation
                            # was scheduled for.
                            proc = job._scheduled_for = fileno_to_inq[ready_fd]
                        except KeyError:
                            # write was scheduled for this fd but the process
                            # has since exited and the message must be sent to
                            # another process.
                            put_message(job)
                            continue
                        cor = _write_job(proc, ready_fd, job)
                        job._writer = ref(cor)
                        mark_write_gen_as_active(cor)
                        mark_write_fd_as_active(ready_fd)
                        mark_worker_as_busy(ready_fd)

                        # Try to write immediately, in case there's an error.
                        try:
                            next(cor)
                        except StopIteration:
                            pass
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                        else:
                            add_writer(ready_fd, cor)
コード例 #25
0
ファイル: asynpool.py プロジェクト: rhayesbite/celery
        def schedule_writes(ready_fds, shuffle=random.shuffle):
            # Schedule write operation to ready file descriptor.
            # The file descriptor is writeable, but that does not
            # mean the process is currently reading from the socket.
            # The socket is buffered so writeable simply means that
            # the buffer can accept at least 1 byte of data.
            shuffle(ready_fds)
            for ready_fd in ready_fds:
                if ready_fd in active_writes:
                    # already writing to this fd
                    continue
                if is_fair_strategy and ready_fd in busy_workers:
                    # worker is already busy with another task
                    continue
                if ready_fd not in all_inqueues:
                    hub_remove(ready_fd)
                    continue
                try:
                    job = pop_message()
                except IndexError:
                    # no more messages, remove all inactive fds from the hub.
                    # this is important since the fds are always writeable
                    # as long as there's 1 byte left in the buffer, and so
                    # this may create a spinloop where the event loop
                    # always wakes up.
                    for inqfd in diff(active_writes):
                        hub_remove(inqfd)
                    break

                else:
                    if not job._accepted:  # job not accepted by another worker
                        try:
                            # keep track of what process the write operation
                            # was scheduled for.
                            proc = job._scheduled_for = fileno_to_inq[ready_fd]
                        except KeyError:
                            # write was scheduled for this fd but the process
                            # has since exited and the message must be sent to
                            # another process.
                            put_message(job)
                            continue
                        cor = _write_job(proc, ready_fd, job)
                        job._writer = ref(cor)
                        mark_write_gen_as_active(cor)
                        mark_write_fd_as_active(ready_fd)
                        mark_worker_as_busy(ready_fd)

                        # Try to write immediately, in case there's an error.
                        try:
                            next(cor)
                        except StopIteration:
                            pass
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                        else:
                            add_writer(ready_fd, cor)
コード例 #26
0
ファイル: base.py プロジェクト: tliron/kombu
 def _read(loop):
     try:
         drain_events(timeout=0)
     except timeout:
         return
     except error as exc:
         if get_errno(exc) in _unavail:
             return
         raise
     loop.call_soon(_read, loop)
コード例 #27
0
ファイル: asynpool.py プロジェクト: subssn21/celery
def _select(readers=None, writers=None, err=None, timeout=0):
    """Simple wrapper to :class:`~select.select`.

    :param readers: Set of reader fds to test if readable.
    :param writers: Set of writer fds to test if writable.
    :param err: Set of fds to test for error condition.

    All fd sets passed must be mutable as this function
    will remove non-working fds from them, this also means
    the caller must make sure there are still fds in the sets
    before calling us again.

    :returns: tuple of ``(readable, writable, again)``, where
        ``readable`` is a set of fds that have data available for read,
        ``writable`` is a set of fds that is ready to be written to
        and ``again`` is a flag that if set means the caller must
        throw away the result and call us again.

    """
    readers = set() if readers is None else readers
    writers = set() if writers is None else writers
    err = set() if err is None else err
    try:
        r, w, e = select.select(readers, writers, err, timeout)
        if e:
            r = list(set(r) | set(e))
        return r, w, 0
    except (select.error, socket.error) as exc:
        if get_errno(exc) == errno.EINTR:
            return [], [], 1
        elif get_errno(exc) in SELECT_BAD_FD:
            for fd in readers | writers | err:
                try:
                    select.select([fd], [], [], 0)
                except (select.error, socket.error) as exc:
                    if get_errno(exc) not in SELECT_BAD_FD:
                        raise
                    readers.discard(fd)
                    writers.discard(fd)
                    err.discard(fd)
            return [], [], 1
        else:
            raise
コード例 #28
0
def _select(readers=None, writers=None, err=None, timeout=0):
    """Simple wrapper to :class:`~select.select`.

    :param readers: Set of reader fds to test if readable.
    :param writers: Set of writer fds to test if writable.
    :param err: Set of fds to test for error condition.

    All fd sets passed must be mutable as this function
    will remove non-working fds from them, this also means
    the caller must make sure there are still fds in the sets
    before calling us again.

    :returns: tuple of ``(readable, writable, again)``, where
        ``readable`` is a set of fds that have data available for read,
        ``writable`` is a set of fds that is ready to be written to
        and ``again`` is a flag that if set means the caller must
        throw away the result and call us again.

    """
    readers = set() if readers is None else readers
    writers = set() if writers is None else writers
    err = set() if err is None else err
    try:
        r, w, e = select.select(readers, writers, err, timeout)
        if e:
            r = list(set(r) | set(e))
        return r, w, 0
    except (select.error, socket.error) as exc:
        if get_errno(exc) == errno.EINTR:
            return [], [], 1
        elif get_errno(exc) in SELECT_BAD_FD:
            for fd in readers | writers | err:
                try:
                    select.select([fd], [], [], 0)
                except (select.error, socket.error) as exc:
                    if get_errno(exc) not in SELECT_BAD_FD:
                        raise
                    readers.discard(fd)
                    writers.discard(fd)
                    err.discard(fd)
            return [], [], 1
        else:
            raise
コード例 #29
0
 def _read(loop):
     if not connection.connected:
         raise ConnectionError('Socket was disconnected')
     try:
         drain_events(timeout=0)
     except timeout:
         return
     except error as exc:
         if get_errno(exc) in _unavail:
             return
         raise
     loop.call_soon(_read, loop)
コード例 #30
0
ファイル: asynpool.py プロジェクト: OnShift/celery
 def _flush_writer(self, writer):
     try:
         while 1:
             try:
                 next(writer)
             except StopIteration:
                 break
     except (OSError, IOError) as exc:
         if get_errno(exc) != errno.EBADF:
             raise
     finally:
         self._active_writers.discard(writer)
コード例 #31
0
ファイル: base.py プロジェクト: jesper/kombu
 def _reader(self, connection, timeout=socket.timeout, error=socket.error,
             get_errno=get_errno, _unavail=(errno.EAGAIN, errno.EINTR)):
     drain_events = connection.drain_events
     while 1:
         try:
             yield drain_events(timeout=0)
         except timeout:
             break
         except error as exc:
             if get_errno(exc) in _unavail:
                 break
             raise
コード例 #32
0
ファイル: base.py プロジェクト: 1stvamp/kombu
 def _read(loop):
     if not connection.connected:
         raise ConnectionError('Socket was disconnected')
     try:
         drain_events(timeout=0)
     except timeout:
         return
     except error as exc:
         if get_errno(exc) in _unavail:
             return
         raise
     loop.call_soon(_read, loop)
コード例 #33
0
ファイル: asynpool.py プロジェクト: OnShift/celery
 def _flush_writer(self, writer):
     try:
         while 1:
             try:
                 next(writer)
             except StopIteration:
                 break
     except (OSError, IOError) as exc:
         if get_errno(exc) != errno.EBADF:
             raise
     finally:
         self._active_writers.discard(writer)
コード例 #34
0
ファイル: platforms.py プロジェクト: anukat2015/AIR
def maybe_drop_privileges(uid=None, gid=None):
    """Change process privileges to new user/group.

    If UID and GID is specified, the real user/group is changed.

    If only UID is specified, the real user is changed, and the group is
    changed to the users primary group.

    If only GID is specified, only the group is changed.

    """
    if sys.platform == 'win32':
        return
    if os.geteuid():
        # no point trying to setuid unless we're root.
        if not os.getuid():
            raise AssertionError('contact support')
    uid = uid and parse_uid(uid)
    gid = gid and parse_gid(gid)

    if uid:
        # If GID isn't defined, get the primary GID of the user.
        if not gid and pwd:
            gid = pwd.getpwuid(uid).pw_gid
        # Must set the GID before initgroups(), as setgid()
        # is known to zap the group list on some platforms.

        # setgid must happen before setuid (otherwise the setgid operation
        # may fail because of insufficient privileges and possibly stay
        # in a privileged group).
        setgid(gid)
        initgroups(uid, gid)

        # at last:
        setuid(uid)
        # ... and make sure privileges cannot be restored:
        try:
            setuid(0)
        except OSError as exc:
            if get_errno(exc) != errno.EPERM:
                raise
            pass  # Good: cannot restore privileges.
        else:
            raise RuntimeError(
                'non-root user able to restore privileges after setuid.')
    else:
        gid and setgid(gid)

    if uid and (not os.getuid()) and not (os.geteuid()):
        raise AssertionError('Still root uid after drop privileges!')
    if gid and (not os.getgid()) and not (os.getegid()):
        raise AssertionError('Still root gid after drop privileges!')
コード例 #35
0
ファイル: platforms.py プロジェクト: akestner/celery
def maybe_drop_privileges(uid=None, gid=None):
    """Change process privileges to new user/group.

    If UID and GID is specified, the real user/group is changed.

    If only UID is specified, the real user is changed, and the group is
    changed to the users primary group.

    If only GID is specified, only the group is changed.

    """
    if sys.platform == 'win32':
        return
    if os.geteuid():
        # no point trying to setuid unless we're root.
        if not os.getuid():
            raise AssertionError('contact support')
    uid = uid and parse_uid(uid)
    gid = gid and parse_gid(gid)

    if uid:
        # If GID isn't defined, get the primary GID of the user.
        if not gid and pwd:
            gid = pwd.getpwuid(uid).pw_gid
        # Must set the GID before initgroups(), as setgid()
        # is known to zap the group list on some platforms.

        # setgid must happen before setuid (otherwise the setgid operation
        # may fail because of insufficient privileges and possibly stay
        # in a privileged group).
        setgid(gid)
        initgroups(uid, gid)

        # at last:
        setuid(uid)
        # ... and make sure privileges cannot be restored:
        try:
            setuid(0)
        except OSError as exc:
            if get_errno(exc) != errno.EPERM:
                raise
            pass  # Good: cannot restore privileges.
        else:
            raise RuntimeError(
                'non-root user able to restore privileges after setuid.')
    else:
        gid and setgid(gid)

    if uid and (not os.getuid()) and not (os.geteuid()):
        raise AssertionError('Still root uid after drop privileges!')
    if gid and (not os.getgid()) and not (os.getegid()):
        raise AssertionError('Still root gid after drop privileges!')
コード例 #36
0
ファイル: hub.py プロジェクト: 1stvamp/kombu
 def fire_timers(self, min_delay=1, max_delay=10, max_timers=10,
                 propagate=()):
     timer = self.timer
     delay = None
     if timer and timer._queue:
         for i in range(max_timers):
             delay, entry = next(self.scheduler)
             if entry is None:
                 break
             try:
                 entry()
             except propagate:
                 raise
             except (MemoryError, AssertionError):
                 raise
             except OSError as exc:
                 if get_errno(exc) == errno.ENOMEM:
                     raise
                 logger.error('Error in timer: %r', exc, exc_info=1)
             except Exception as exc:
                 logger.error('Error in timer: %r', exc, exc_info=1)
     return min(max(delay or 0, min_delay), max_delay)
コード例 #37
0
ファイル: consumer.py プロジェクト: jamesdear/celery
 def start(self):
     blueprint, loop = self.blueprint, self.loop
     while blueprint.state != CLOSE:
         self.restart_count += 1
         maybe_shutdown()
         try:
             blueprint.start(self)
         except self.connection_errors as exc:
             if isinstance(exc, OSError) and get_errno(exc) == errno.EMFILE:
                 raise  # Too many open files
             maybe_shutdown()
             try:
                 self._restart_state.step()
             except RestartFreqExceeded as exc:
                 crit('Frequent restarts detected: %r', exc, exc_info=1)
                 sleep(1)
             if blueprint.state != CLOSE and self.connection:
                 warn(CONNECTION_RETRY, exc_info=True)
                 try:
                     self.connection.collect()
                 except Exception:
                     pass
                 self.on_close()
                 blueprint.restart(self)
コード例 #38
0
 def start(self):
     blueprint, loop = self.blueprint, self.loop
     while blueprint.state != CLOSE:
         self.restart_count += 1
         maybe_shutdown()
         try:
             blueprint.start(self)
         except self.connection_errors as exc:
             if isinstance(exc, OSError) and get_errno(exc) == errno.EMFILE:
                 raise  # Too many open files
             maybe_shutdown()
             try:
                 self._restart_state.step()
             except RestartFreqExceeded as exc:
                 crit('Frequent restarts detected: %r', exc, exc_info=1)
                 sleep(1)
             if blueprint.state != CLOSE and self.connection:
                 warn(CONNECTION_RETRY, exc_info=True)
                 try:
                     self.connection.collect()
                 except Exception:
                     pass
                 self.on_close()
                 blueprint.restart(self)
コード例 #39
0
ファイル: hub.py プロジェクト: 1995rishi/flaskmap
    def create_loop(self,
                    generator=generator, sleep=sleep, min=min, next=next,
                    Empty=Empty, StopIteration=StopIteration,
                    KeyError=KeyError, READ=READ, WRITE=WRITE, ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        todo = self._ready
        propagate = self.propagate_errors

        while 1:
            for tick_callback in on_tick:
                tick_callback()

            while todo:
                item = todo.popleft()
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fd, event in events or ():
                    general_error = False
                    if fd in consolidate and \
                            writers.get(fd) is None:
                        to_consolidate.append(fd)
                        continue
                    cb = cbargs = None

                    if event & READ:
                        try:
                            cb, cbargs = readers[fd]
                        except KeyError:
                            self.remove_reader(fd)
                            continue
                    elif event & WRITE:
                        try:
                            cb, cbargs = writers[fd]
                        except KeyError:
                            self.remove_writer(fd)
                            continue
                    elif event & ERR:
                        general_error = True
                    else:
                        logger.info(W_UNKNOWN_EVENT, event, fd)
                        general_error = True

                    if general_error:
                        try:
                            cb, cbargs = (readers.get(fd) or
                                          writers.get(fd))
                        except TypeError:
                            pass

                    if cb is None:
                        self.remove(fd)
                        continue

                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                            hub_remove(fd)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fd)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
コード例 #40
0
ファイル: hub.py プロジェクト: saintland/SADI-Galaxy-Docker
    def create_loop(self,
                    generator=generator,
                    sleep=sleep,
                    min=min,
                    next=next,
                    Empty=Empty,
                    StopIteration=StopIteration,
                    KeyError=KeyError,
                    READ=READ,
                    WRITE=WRITE,
                    ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        todo = self._ready
        propagate = self.propagate_errors

        while 1:
            for tick_callback in on_tick:
                tick_callback()

            while todo:
                item = todo.popleft()
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            #print('[[[HUB]]]: %s' % (self.repr_active(), ))
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                    #print('[EVENTS]: %s' % (self.nepr_events(events or []), ))
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fileno, event in events or ():
                    if fileno in consolidate and \
                            writers.get(fileno) is None:
                        to_consolidate.append(fileno)
                        continue
                    cb = cbargs = None
                    try:
                        if event & READ:
                            cb, cbargs = readers[fileno]
                        elif event & WRITE:
                            cb, cbargs = writers[fileno]
                        elif event & ERR:
                            try:
                                cb, cbargs = (readers.get(fileno)
                                              or writers.get(fileno))
                            except TypeError:
                                pass
                    except (KeyError, Empty):
                        hub_remove(fileno)
                        continue
                    if cb is None:
                        continue
                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                            hub_remove(fileno)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fileno)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
コード例 #41
0
ファイル: hub.py プロジェクト: 571451370/devstack_mitaka
    def create_loop(self,
                    generator=generator,
                    sleep=sleep,
                    min=min,
                    next=next,
                    Empty=Empty,
                    StopIteration=StopIteration,
                    KeyError=KeyError,
                    READ=READ,
                    WRITE=WRITE,
                    ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        todo = self._ready
        propagate = self.propagate_errors

        while 1:
            for tick_callback in on_tick:
                tick_callback()

            while todo:
                item = todo.popleft()
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fd, event in events or ():
                    general_error = False
                    if fd in consolidate and \
                            writers.get(fd) is None:
                        to_consolidate.append(fd)
                        continue
                    cb = cbargs = None

                    if event & READ:
                        try:
                            cb, cbargs = readers[fd]
                        except KeyError:
                            self.remove_reader(fd)
                            continue
                    elif event & WRITE:
                        try:
                            cb, cbargs = writers[fd]
                        except KeyError:
                            self.remove_writer(fd)
                            continue
                    elif event & ERR:
                        general_error = True
                    else:
                        logger.info(W_UNKNOWN_EVENT, event, fd)
                        general_error = True

                    if general_error:
                        try:
                            cb, cbargs = (readers.get(fd) or writers.get(fd))
                        except TypeError:
                            pass

                    if cb is None:
                        self.remove(fd)
                        continue

                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                            hub_remove(fd)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fd)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
コード例 #42
0
        def schedule_writes(ready_fds, curindex=[0]):
            # Schedule write operation to ready file descriptor.
            # The file descriptor is writeable, but that does not
            # mean the process is currently reading from the socket.
            # The socket is buffered so writeable simply means that
            # the buffer can accept at least 1 byte of data.

            # This means we have to cycle between the ready fds.
            # the first version used shuffle, but using i % total
            # is about 30% faster with many processes.  The latter
            # also shows more fairness in write stats when used with
            # many processes [XXX On OS X, this may vary depending
            # on event loop implementation (i.e select vs epoll), so
            # have to test further]
            total = len(ready_fds)

            for i in range(total):
                ready_fd = ready_fds[curindex[0] % total]
                if ready_fd in active_writes:
                    # already writing to this fd
                    curindex[0] += 1
                    continue
                if is_fair_strategy and ready_fd in busy_workers:
                    # worker is already busy with another task
                    curindex[0] += 1
                    continue
                if ready_fd not in all_inqueues:
                    hub_remove(ready_fd)
                    curindex[0] += 1
                    continue
                try:
                    job = pop_message()
                except IndexError:
                    # no more messages, remove all inactive fds from the hub.
                    # this is important since the fds are always writeable
                    # as long as there's 1 byte left in the buffer, and so
                    # this may create a spinloop where the event loop
                    # always wakes up.
                    for inqfd in diff(active_writes):
                        hub_remove(inqfd)
                    break
                else:
                    if not job._accepted:  # job not accepted by another worker
                        try:
                            # keep track of what process the write operation
                            # was scheduled for.
                            proc = job._scheduled_for = fileno_to_inq[ready_fd]
                        except KeyError:
                            # write was scheduled for this fd but the process
                            # has since exited and the message must be sent to
                            # another process.
                            put_back_message(job)
                            curindex[0] += 1
                            continue
                        cor = _write_job(proc, ready_fd, job)
                        job._writer = ref(cor)
                        mark_write_gen_as_active(cor)
                        mark_write_fd_as_active(ready_fd)
                        mark_worker_as_busy(ready_fd)

                        # Try to write immediately, in case there's an error.
                        try:
                            next(cor)
                        except StopIteration:
                            pass
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                        else:
                            add_writer(ready_fd, cor)
                curindex[0] += 1
コード例 #43
0
ファイル: hub.py プロジェクト: 1stvamp/kombu
    def create_loop(self,
                    generator=generator, sleep=sleep, min=min, next=next,
                    Empty=Empty, StopIteration=StopIteration,
                    KeyError=KeyError, READ=READ, WRITE=WRITE, ERR=ERR):
        readers, writers = self.readers, self.writers
        poll = self.poller.poll
        fire_timers = self.fire_timers
        hub_remove = self.remove
        scheduled = self.timer._queue
        consolidate = self.consolidate
        consolidate_callback = self.consolidate_callback
        on_tick = self.on_tick
        todo = self._ready
        propagate = self.propagate_errors

        while 1:
            for tick_callback in on_tick:
                tick_callback()

            while todo:
                item = todo.popleft()
                if item:
                    item()

            poll_timeout = fire_timers(propagate=propagate) if scheduled else 1
            #print('[[[HUB]]]: %s' % (self.repr_active(), ))
            if readers or writers:
                to_consolidate = []
                try:
                    events = poll(poll_timeout)
                    #print('[EVENTS]: %s' % (self.repr_events(events or []), ))
                except ValueError:  # Issue 882
                    raise StopIteration()

                for fileno, event in events or ():
                    if fileno in consolidate and \
                            writers.get(fileno) is None:
                        to_consolidate.append(fileno)
                        continue
                    cb = cbargs = None
                    try:
                        if event & READ:
                            cb, cbargs = readers[fileno]
                        elif event & WRITE:
                            cb, cbargs = writers[fileno]
                        elif event & ERR:
                            try:
                                cb, cbargs = (readers.get(fileno) or
                                              writers.get(fileno))
                            except TypeError:
                                pass
                    except (KeyError, Empty):
                        hub_remove(fileno)
                        continue
                    if cb is None:
                        continue
                    if isinstance(cb, generator):
                        try:
                            next(cb)
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                            hub_remove(fileno)
                        except StopIteration:
                            pass
                        except Exception:
                            hub_remove(fileno)
                            raise
                    else:
                        try:
                            cb(*cbargs)
                        except Empty:
                            pass
                if to_consolidate:
                    consolidate_callback(to_consolidate)
            else:
                # no sockets yet, startup is probably not done.
                sleep(min(poll_timeout, 0.1))
            yield
コード例 #44
0
ファイル: asynpool.py プロジェクト: NymphMAX/Arianrhod
        def schedule_writes(ready_fds, curindex=[0]):
            # Schedule write operation to ready file descriptor.
            # The file descriptor is writeable, but that does not
            # mean the process is currently reading from the socket.
            # The socket is buffered so writeable simply means that
            # the buffer can accept at least 1 byte of data.

            # This means we have to cycle between the ready fds.
            # the first version used shuffle, but using i % total
            # is about 30% faster with many processes.  The latter
            # also shows more fairness in write stats when used with
            # many processes [XXX On OS X, this may vary depending
            # on event loop implementation (i.e select vs epoll), so
            # have to test further]
            total = len(ready_fds)

            for i in range(total):
                ready_fd = ready_fds[curindex[0] % total]
                if ready_fd in active_writes:
                    # already writing to this fd
                    curindex[0] += 1
                    continue
                if is_fair_strategy and ready_fd in busy_workers:
                    # worker is already busy with another task
                    curindex[0] += 1
                    continue
                if ready_fd not in all_inqueues:
                    hub_remove(ready_fd)
                    curindex[0] += 1
                    continue
                try:
                    job = pop_message()
                except IndexError:
                    # no more messages, remove all inactive fds from the hub.
                    # this is important since the fds are always writeable
                    # as long as there's 1 byte left in the buffer, and so
                    # this may create a spinloop where the event loop
                    # always wakes up.
                    for inqfd in diff(active_writes):
                        hub_remove(inqfd)
                    break
                else:
                    if not job._accepted:  # job not accepted by another worker
                        try:
                            # keep track of what process the write operation
                            # was scheduled for.
                            proc = job._scheduled_for = fileno_to_inq[ready_fd]
                        except KeyError:
                            # write was scheduled for this fd but the process
                            # has since exited and the message must be sent to
                            # another process.
                            put_back_message(job)
                            curindex[0] += 1
                            continue
                        cor = _write_job(proc, ready_fd, job)
                        job._writer = ref(cor)
                        mark_write_gen_as_active(cor)
                        mark_write_fd_as_active(ready_fd)
                        mark_worker_as_busy(ready_fd)

                        # Try to write immediately, in case there's an error.
                        try:
                            next(cor)
                        except StopIteration:
                            pass
                        except OSError as exc:
                            if get_errno(exc) != errno.EBADF:
                                raise
                        else:
                            add_writer(ready_fd, cor)
                curindex[0] += 1