Пример #1
0
    def deserialize_safe(cls, byts, protocol_version):
        proto_version = max(3, protocol_version)
        p = 0
        values = []
        for col_type in cls.subtypes:
            if p == len(byts):
                break
            itemlen = int32_unpack(byts[p:p + 4])
            p += 4
            if itemlen >= 0:
                item = byts[p:p + itemlen]
                p += itemlen
            else:
                item = None
            # collections inside UDTs are always encoded with at least the
            # version 3 format
            values.append(col_type.from_binary(item, proto_version))

        if len(values) < len(cls.subtypes):
            nones = [None] * (len(cls.subtypes) - len(values))
            values = values + nones

        if cls.mapped_class:
            return cls.mapped_class(**dict(zip(cls.fieldnames, values)))
        else:
            return cls.tuple_type(*values)
    def process_io_buffer(self):
        while True:
            pos = self._iobuf.tell()
            if pos < self._full_header_length or (
                    self._total_reqd_bytes > 0
                    and pos < self._total_reqd_bytes):
                # we don't have a complete header yet or we
                # already saw a header, but we don't have a
                # complete message yet
                return
            else:
                # have enough for header, read body len from header
                self._iobuf.seek(self._header_length)
                body_len = int32_unpack(self._iobuf.read(4))

                # seek to end to get length of current buffer
                self._iobuf.seek(0, os.SEEK_END)
                pos = self._iobuf.tell()

                if pos >= body_len + self._full_header_length:
                    # read message header and body
                    self._iobuf.seek(0)
                    msg = self._iobuf.read(self._full_header_length + body_len)

                    # leave leftover in current buffer
                    leftover = self._iobuf.read()
                    self._iobuf = io.BytesIO()
                    self._iobuf.write(leftover)

                    self._total_reqd_bytes = 0
                    self.process_msg(msg, body_len)
                else:
                    self._total_reqd_bytes = body_len + self._full_header_length
                    return
Пример #3
0
    def handle_read(self):
        try:
            buf = self.recv(self.in_buffer_size)
        except socket.error as err:
            if err.args[0] not in NONBLOCKING:
                self.defunct(err)
            return

        if buf:
            self._buf += buf
            while True:
                if len(self._buf) < 8:
                    # we don't have a complete header yet
                    break
                elif self._total_reqd_bytes and len(self._buf) < self._total_reqd_bytes:
                    # we already saw a header, but we don't have a complete message yet
                    break
                else:
                    body_len = int32_unpack(self._buf[4:8])
                    if len(self._buf) - 8 >= body_len:
                        msg = self._buf[:8 + body_len]
                        self._buf = self._buf[8 + body_len:]
                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8

            if not self._callbacks:
                self._readable = False
        else:
            self.close()
Пример #4
0
    def deserialize_safe(cls, byts, protocol_version):
        proto_version = max(3, protocol_version)
        p = 0
        values = []
        for col_type in cls.subtypes:
            if p == len(byts):
                break
            itemlen = int32_unpack(byts[p:p + 4])
            p += 4
            if itemlen >= 0:
                item = byts[p:p + itemlen]
                p += itemlen
            else:
                item = None
            # collections inside UDTs are always encoded with at least the
            # version 3 format
            values.append(col_type.from_binary(item, proto_version))

        if len(values) < len(cls.subtypes):
            nones = [None] * (len(cls.subtypes) - len(values))
            values = values + nones

        if cls.mapped_class:
            return cls.mapped_class(**dict(zip(cls.fieldnames, values)))
        else:
            return cls.tuple_type(*values)
Пример #5
0
    def process_io_buffer(self):
        while True:
            pos = self._iobuf.tell()
            if pos < self._full_header_length or (self._total_reqd_bytes > 0 and pos < self._total_reqd_bytes):
                # we don't have a complete header yet or we
                # already saw a header, but we don't have a
                # complete message yet
                return
            else:
                # have enough for header, read body len from header
                self._iobuf.seek(self._header_length)
                body_len = int32_unpack(self._iobuf.read(4))

                # seek to end to get length of current buffer
                self._iobuf.seek(0, os.SEEK_END)
                pos = self._iobuf.tell()

                if pos >= body_len + self._full_header_length:
                    # read message header and body
                    self._iobuf.seek(0)
                    msg = self._iobuf.read(self._full_header_length + body_len)

                    # leave leftover in current buffer
                    leftover = self._iobuf.read()
                    self._iobuf = io.BytesIO()
                    self._iobuf.write(leftover)

                    self._total_reqd_bytes = 0
                    self.process_msg(msg, body_len)
                else:
                    self._total_reqd_bytes = body_len + self._full_header_length
                    return
Пример #6
0
    def handle_read(self, watcher, revents, errno=None):
        if revents & libev.EV_ERROR:
            if errno:
                exc = IOError(errno, os.strerror(errno))
            else:
                exc = Exception("libev reported an error")

            self.defunct(exc)
            return
        try:
            while True:
                buf = self._socket.recv(self.in_buffer_size)
                self._iobuf.write(buf)
                if len(buf) < self.in_buffer_size:
                    break
        except socket.error as err:
            if ssl and isinstance(err, ssl.SSLError):
                if err.args[0] not in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE):
                    self.defunct(err)
                    return
            elif err.args[0] not in NONBLOCKING:
                self.defunct(err)
                return

        if self._iobuf.tell():
            while True:
                pos = self._iobuf.tell()
                if pos < 8 or (self._total_reqd_bytes > 0 and pos < self._total_reqd_bytes):
                    # we don't have a complete header yet or we
                    # already saw a header, but we don't have a
                    # complete message yet
                    break
                else:
                    # have enough for header, read body len from header
                    self._iobuf.seek(4)
                    body_len = int32_unpack(self._iobuf.read(4))

                    # seek to end to get length of current buffer
                    self._iobuf.seek(0, os.SEEK_END)
                    pos = self._iobuf.tell()

                    if pos >= body_len + 8:
                        # read message header and body
                        self._iobuf.seek(0)
                        msg = self._iobuf.read(8 + body_len)

                        # leave leftover in current buffer
                        leftover = self._iobuf.read()
                        self._iobuf = StringIO()
                        self._iobuf.write(leftover)

                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8
                        break
        else:
            log.debug("Connection %s closed by server", self)
            self.close()
Пример #7
0
    def handle_read(self):
        run_select = partial(select.select, (self._socket, ), (), ())
        while True:
            try:
                run_select()
            except Exception as exc:
                if not self.is_closed:
                    log.debug("Exception during read select() for %s: %s",
                              self, exc)
                    self.defunct(exc)
                return

            try:
                buf = self._socket.recv(self.in_buffer_size)
                self._iobuf.write(buf)
            except socket.error as err:
                if not is_timeout(err):
                    log.debug("Exception during socket recv for %s: %s", self,
                              err)
                    self.defunct(err)
                    return  # leave the read loop

            if self._iobuf.tell():
                while True:
                    pos = self._iobuf.tell()
                    if pos < 8 or (self._total_reqd_bytes > 0
                                   and pos < self._total_reqd_bytes):
                        # we don't have a complete header yet or we
                        # already saw a header, but we don't have a
                        # complete message yet
                        break
                    else:
                        # have enough for header, read body len from header
                        self._iobuf.seek(4)
                        body_len = int32_unpack(self._iobuf.read(4))

                        # seek to end to get length of current buffer
                        self._iobuf.seek(0, os.SEEK_END)
                        pos = self._iobuf.tell()

                        if pos >= body_len + 8:
                            # read message header and body
                            self._iobuf.seek(0)
                            msg = self._iobuf.read(8 + body_len)

                            # leave leftover in current buffer
                            leftover = self._iobuf.read()
                            self._iobuf = StringIO()
                            self._iobuf.write(leftover)

                            self._total_reqd_bytes = 0
                            self.process_msg(msg, body_len)
                        else:
                            self._total_reqd_bytes = body_len + 8
                            break
            else:
                log.debug("connection closed by server")
                self.close()
                return
Пример #8
0
    def handle_read(self):
        run_select = partial(select.select, (self._socket,), (), ())
        while True:
            try:
                run_select()
            except Exception as exc:
                if not self.is_closed:
                    log.debug(
                        "Exception during read select() for %s: %s", self, exc)
                    self.defunct(exc)
                return

            try:
                buf = self._socket.recv(self.in_buffer_size)
                self._iobuf.write(buf)
            except socket.error as err:
                if not is_timeout(err):
                    log.debug(
                        "Exception during socket recv for %s: %s", self, err)
                    self.defunct(err)
                    return  # leave the read loop

            if self._iobuf.tell():
                while True:
                    pos = self._iobuf.tell()
                    if pos < 8 or (self._total_reqd_bytes > 0
                                   and pos < self._total_reqd_bytes):
                        # we don't have a complete header yet or we
                        # already saw a header, but we don't have a
                        # complete message yet
                        break
                    else:
                        # have enough for header, read body len from header
                        self._iobuf.seek(4)
                        body_len = int32_unpack(self._iobuf.read(4))

                        # seek to end to get length of current buffer
                        self._iobuf.seek(0, os.SEEK_END)
                        pos = self._iobuf.tell()

                        if pos >= body_len + 8:
                            # read message header and body
                            self._iobuf.seek(0)
                            msg = self._iobuf.read(8 + body_len)

                            # leave leftover in current buffer
                            leftover = self._iobuf.read()
                            self._iobuf = StringIO()
                            self._iobuf.write(leftover)

                            self._total_reqd_bytes = 0
                            self.process_msg(msg, body_len)
                        else:
                            self._total_reqd_bytes = body_len + 8
                            break
            else:
                log.debug("connection closed by server")
                self.close()
                return
Пример #9
0
    def handle_read(self):
        try:
            while True:
                buf = self.recv(self.in_buffer_size)
                self._iobuf.write(buf)
                if len(buf) < self.in_buffer_size:
                    break
        except socket.error as err:
            if ssl and isinstance(err, ssl.SSLError):
                if err.args[0] not in (ssl.SSL_ERROR_WANT_READ,
                                       ssl.SSL_ERROR_WANT_WRITE):
                    self.defunct(err)
                    return
            elif err.args[0] not in NONBLOCKING:
                self.defunct(err)
                return

        if self._iobuf.tell():
            while True:
                pos = self._iobuf.tell()
                if pos < 8 or (self._total_reqd_bytes > 0
                               and pos < self._total_reqd_bytes):
                    # we don't have a complete header yet or we
                    # already saw a header, but we don't have a
                    # complete message yet
                    break
                else:
                    # have enough for header, read body len from header
                    self._iobuf.seek(4)
                    body_len = int32_unpack(self._iobuf.read(4))

                    # seek to end to get length of current buffer
                    self._iobuf.seek(0, os.SEEK_END)
                    pos = self._iobuf.tell()

                    if pos >= body_len + 8:
                        # read message header and body
                        self._iobuf.seek(0)
                        msg = self._iobuf.read(8 + body_len)

                        # leave leftover in current buffer
                        leftover = self._iobuf.read()
                        self._iobuf = BytesIO()
                        self._iobuf.write(leftover)

                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8
                        break

            if not self._callbacks and not self.is_control_connection:
                self._readable = False
    def handle_read(self):
        try:
            while True:
                buf = self.recv(self.in_buffer_size)
                self._iobuf.write(buf)
                if len(buf) < self.in_buffer_size:
                    break
        except socket.error as err:
            if ssl and isinstance(err, ssl.SSLError):
                if err.args[0] not in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE):
                    self.defunct(err)
                    return
            elif err.args[0] not in NONBLOCKING:
                self.defunct(err)
                return

        if self._iobuf.tell():
            while True:
                pos = self._iobuf.tell()
                if pos < 8 or (self._total_reqd_bytes > 0 and pos < self._total_reqd_bytes):
                    # we don't have a complete header yet or we
                    # already saw a header, but we don't have a
                    # complete message yet
                    break
                else:
                    # have enough for header, read body len from header
                    self._iobuf.seek(4)
                    body_len = int32_unpack(self._iobuf.read(4))

                    # seek to end to get length of current buffer
                    self._iobuf.seek(0, os.SEEK_END)
                    pos = self._iobuf.tell()

                    if pos >= body_len + 8:
                        # read message header and body
                        self._iobuf.seek(0)
                        msg = self._iobuf.read(8 + body_len)

                        # leave leftover in current buffer
                        leftover = self._iobuf.read()
                        self._iobuf = BytesIO()
                        self._iobuf.write(leftover)

                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8
                        break

            if not self._callbacks and not self.is_control_connection:
                self._readable = False
Пример #11
0
    def handle_read(self):
        try:
            while True:
                buf = self.recv(self.in_buffer_size)
                self._iobuf.write(buf)
                if len(buf) < self.in_buffer_size:
                    break
        except socket.error as err:
            if err.args[0] not in NONBLOCKING:
                self.defunct(err)
            return

        if self._iobuf.tell():
            while True:
                pos = self._iobuf.tell()
                if pos < 8 or (self._total_reqd_bytes > 0 and pos < self._total_reqd_bytes):
                    # we don't have a complete header yet or we
                    # already saw a header, but we don't have a
                    # complete message yet
                    break
                else:
                    self._iobuf.seek(4)
                    body_len = int32_unpack(self._iobuf.read(4))

                    # seek to end to get length of current buffer
                    self._iobuf.seek(0, os.SEEK_END)
                    pos = self._iobuf.tell()

                    if pos >= body_len + 8:
                        # read message header and body
                        self._iobuf.seek(0)
                        msg = self._iobuf.read(8 + body_len)

                        # leave leftover in current buffer
                        leftover = self._iobuf.read()
                        self._iobuf = StringIO()
                        self._iobuf.write(leftover)

                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8
                        break

            if not self._callbacks:
                self._readable = False
        else:
            self.close()
Пример #12
0
    def handle_read(self, watcher, revents):
        try:
            buf = self._socket.recv(self.in_buffer_size)
        except socket.error as err:
            if err.args[0] not in NONBLOCKING:
                self.defunct(err)
            return

        if buf:
            self._iobuf.write(buf)
            while True:
                pos = self._iobuf.tell()
                if pos < 8 or (self._total_reqd_bytes > 0
                               and pos < self._total_reqd_bytes):
                    # we don't have a complete header yet or we
                    # already saw a header, but we don't have a
                    # complete message yet
                    break
                else:
                    # have enough for header, read body len from header
                    self._iobuf.seek(4)
                    body_len_bytes = self._iobuf.read(4)
                    body_len = int32_unpack(body_len_bytes)

                    # seek to end to get length of current buffer
                    self._iobuf.seek(0, os.SEEK_END)
                    pos = self._iobuf.tell()

                    if pos - 8 >= body_len:
                        # read message header and body
                        self._iobuf.seek(0)
                        msg = self._iobuf.read(8 + body_len)

                        # leave leftover in current buffer
                        leftover = self._iobuf.read()
                        self._iobuf = StringIO()
                        self._iobuf.write(leftover)

                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8
                        break
        else:
            log.debug("connection closed by server")
            self.close()
Пример #13
0
    def handle_read(self):
        try:
            while True:
                buf = self.recv(self.in_buffer_size)
                self._iobuf.write(buf)
                if len(buf) < self.in_buffer_size:
                    break
        except socket.error as err:
            if err.args[0] not in NONBLOCKING:
                self.defunct(err)
            return

        if self._iobuf.tell():
            while True:
                pos = self._iobuf.tell()
                if pos < 8 or (self._total_reqd_bytes > 0 and pos < self._total_reqd_bytes):
                    # we don't have a complete header yet or we
                    # already saw a header, but we don't have a
                    # complete message yet
                    break
                else:
                    self._iobuf.seek(4)
                    body_len = int32_unpack(self._iobuf.read(4))

                    # seek to end to get length of current buffer
                    self._iobuf.seek(0, os.SEEK_END)
                    pos = self._iobuf.tell()

                    if pos >= body_len + 8:
                        # read message header and body
                        self._iobuf.seek(0)
                        msg = self._iobuf.read(8 + body_len)

                        # leave leftover in current buffer
                        leftover = self._iobuf.read()
                        self._iobuf = StringIO()
                        self._iobuf.write(leftover)

                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8
                        break

            if not self._callbacks:
                self._readable = False
Пример #14
0
    def handle_read(self, watcher, revents):
        try:
            buf = self._socket.recv(self.in_buffer_size)
        except socket.error as err:
            if err.args[0] not in NONBLOCKING:
                self.defunct(err)
            return

        if buf:
            self._iobuf.write(buf)
            while True:
                pos = self._iobuf.tell()
                if pos < 8 or (self._total_reqd_bytes > 0 and pos < self._total_reqd_bytes):
                    # we don't have a complete header yet or we
                    # already saw a header, but we don't have a
                    # complete message yet
                    break
                else:
                    # have enough for header, read body len from header
                    self._iobuf.seek(4)
                    body_len_bytes = self._iobuf.read(4)
                    body_len = int32_unpack(body_len_bytes)

                    # seek to end to get length of current buffer
                    self._iobuf.seek(0, os.SEEK_END)
                    pos = self._iobuf.tell()

                    if pos - 8 >= body_len:
                        # read message header and body
                        self._iobuf.seek(0)
                        msg = self._iobuf.read(8 + body_len)

                        # leave leftover in current buffer
                        leftover = self._iobuf.read()
                        self._iobuf = StringIO()
                        self._iobuf.write(leftover)

                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8
                        break
        else:
            log.debug("connection closed by server")
            self.close()
Пример #15
0
    def deserialize_safe(cls, byts):
        p = 0
        Result = collections.namedtuple(cls.typename, cls.fieldnames)
        result = []
        for col_type in cls.subtypes:
            if p == len(byts):
                break
            itemlen = int32_unpack(byts[p:p + cls.FIELD_LENGTH])
            p += cls.FIELD_LENGTH
            item = byts[p:p + itemlen]
            p += itemlen
            result.append(col_type.from_binary(item))

        if len(result) < len(cls.subtypes):
            nones = [None] * (len(cls.subtypes) - len(result))
            result = result + nones

        return Result(*result)
Пример #16
0
    def deserialize_safe(cls, byts, protocol_version):
        proto_version = max(3, protocol_version)
        p = 0
        values = []
        for col_type in cls.subtypes:
            if p == len(byts):
                break
            itemlen = int32_unpack(byts[p:p + 4])
            p += 4
            item = byts[p:p + itemlen]
            p += itemlen
            # collections inside UDTs are always encoded with at least the
            # version 3 format
            values.append(col_type.from_binary(item, proto_version))

        if len(values) < len(cls.subtypes):
            nones = [None] * (len(cls.subtypes) - len(values))
            values = values + nones

        return tuple(values)
Пример #17
0
    def deserialize_safe(cls, byts, protocol_version):
        proto_version = max(3, protocol_version)
        p = 0
        values = []
        for col_type in cls.subtypes:
            if p == len(byts):
                break
            itemlen = int32_unpack(byts[p:p + 4])
            p += 4
            item = byts[p:p + itemlen]
            p += itemlen
            # collections inside UDTs are always encoded with at least the
            # version 3 format
            values.append(col_type.from_binary(item, proto_version))

        if len(values) < len(cls.subtypes):
            nones = [None] * (len(cls.subtypes) - len(values))
            values = values + nones

        return tuple(values)
Пример #18
0
 def deserialize(byts):
     scale = int32_unpack(byts[:4])
     unscaled = varint_unpack(byts[4:])
     return Decimal('%de%d' % (unscaled, -scale))
Пример #19
0
def read_int(f):
    return int32_unpack(f.read(4))
Пример #20
0
 def deserialize(byts):
     scale = int32_unpack(byts[:4])
     unscaled = varint_unpack(byts[4:])
     return Decimal('%de%d' % (unscaled, -scale))
Пример #21
0
def read_int(f):
    return int32_unpack(f.read(4))
Пример #22
0
 def deserialize(byts, protocol_version):
     return int32_unpack(byts)
Пример #23
0
 def deserialize(byts, protocol_version):
     return int32_unpack(byts)
Пример #24
0
 def deserialize(byts, protocol_version):
     scale = int32_unpack(byts[:4])
     unscaled = varint_unpack(byts[4:])
     return Decimal("%de%d" % (unscaled, -scale))
Пример #25
0
            if err.args[0] not in NONBLOCKING:
                self.defunct(err)
            return

        if buf:
            self._buf += buf
            while True:
                if len(self._buf) < 8:
                    # we don't have a complete header yet
                    break
                elif self._total_reqd_bytes and len(
                        self._buf) < self._total_reqd_bytes:
                    # we already saw a header, but we don't have a complete message yet
                    break
                else:
                    body_len = int32_unpack(self._buf[4:8])
                    if len(self._buf) - 8 >= body_len:
                        msg = self._buf[:8 + body_len]
                        self._buf = self._buf[8 + body_len:]
                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8
        else:
            log.debug("connection closed by server")
            self.close()

    def handle_pushed(self, response):
        log.debug("Message pushed from server: %r", response)
        for cb in self._push_watchers.get(response.event_type, []):
            try:
Пример #26
0
        except socket.error, err:
            if err.args[0] not in NONBLOCKING:
                self.defunct(err)
            return

        if buf:
            self._buf += buf
            while True:
                if len(self._buf) < 8:
                    # we don't have a complete header yet
                    break
                elif self._total_reqd_bytes and len(self._buf) < self._total_reqd_bytes:
                    # we already saw a header, but we don't have a complete message yet
                    break
                else:
                    body_len = int32_unpack(self._buf[4:8])
                    if len(self._buf) - 8 >= body_len:
                        msg = self._buf[:8 + body_len]
                        self._buf = self._buf[8 + body_len:]
                        self._total_reqd_bytes = 0
                        self.process_msg(msg, body_len)
                    else:
                        self._total_reqd_bytes = body_len + 8
        else:
            log.debug("connection closed by server")
            self.close()

    def handle_pushed(self, response):
        log.debug("Message pushed from server: %r", response)
        for cb in self._push_watchers.get(response.event_type, []):
            try:
Пример #27
0
        if buf:
            self._iobuf.write(buf)
            while True:
                pos = self._iobuf.tell()
                if pos < 8:
                    # we don't have a complete header yet
                    break
                elif self._total_reqd_bytes and self._iobuf.tell() < self._total_reqd_bytes:
                    # we already saw a header, but we don't have a complete message yet
                    break
                else:
                    # have enough for header, read body len from header
                    self._iobuf.seek(4)
                    body_len_bytes = self._iobuf.read(4)
                    body_len = int32_unpack(body_len_bytes)

                    # seek to end to get length of current buffer
                    self._iobuf.seek(0, 2) 
                    pos = self._iobuf.tell() 

                    if pos - 8 >= body_len:
                        # read message header/body
                        self._iobuf.seek(0)
                        msg = self._iobuf.read(8 + body_len)

                        # leave leftover in current buffer
                        self._iobuf = cStringIO.StringIO()
                        self._iobuf.write(self._iobuf.read())

                        self._total_reqd_bytes = 0