def serialize_safe(cls, val, protocol_version): proto_version = max(3, protocol_version) buf = io.BytesIO() for fieldname, subtype in zip(cls.fieldnames, cls.subtypes): item = getattr(val, fieldname) if item is not None: packed_item = subtype.to_binary(getattr(val, fieldname), proto_version) buf.write(int32_pack(len(packed_item))) buf.write(packed_item) else: buf.write(int32_pack(-1)) return buf.getvalue()
def serialize_safe(cls, val, protocol_version): if len(val) > len(cls.subtypes): raise ValueError("Expected %d items in a tuple, but got %d: %s" % (len(cls.subtypes), len(val), val)) proto_version = max(3, protocol_version) buf = io.BytesIO() for item, subtype in zip(val, cls.subtypes): if item is not None: packed_item = subtype.to_binary(item, proto_version) buf.write(int32_pack(len(packed_item))) buf.write(packed_item) else: buf.write(int32_pack(-1)) return buf.getvalue()
def _check_error_recovery_on_buffer_size(self, error_code): c = self.test_successful_connection() header = six.b('\x00\x00\x00\x00') + int32_pack(20000) responses = [ header + (six.b('a') * (4096 - len(header))), six.b('a') * 4096, socket_error(error_code), six.b('a') * 100, socket_error(error_code)] def side_effect(*args): response = responses.pop(0) log.debug('about to mock return {}'.format(response)) if isinstance(response, socket_error): raise response else: return response self.get_socket(c).recv.side_effect = side_effect c.handle_read(*self.null_handle_function_args) # the EAGAIN prevents it from reading the last 100 bytes c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() self.assertEqual(pos, 4096 + 4096) # now tell it to read the last 100 bytes c.handle_read(*self.null_handle_function_args) c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() self.assertEqual(pos, 4096 + 4096 + 100)
def test_egain_on_buffer_size(self, *args): # get a connection that's already fully started c = self.test_successful_connection() header = '\x00\x00\x00\x00' + int32_pack(20000) responses = [ header + ('a' * (4096 - len(header))), 'a' * 4096, socket_error(errno.EAGAIN), 'a' * 100, socket_error(errno.EAGAIN) ] def side_effect(*args): response = responses.pop(0) if isinstance(response, socket_error): raise response else: return response c._socket.recv.side_effect = side_effect c.handle_read(None, 0) self.assertEquals(c._total_reqd_bytes, 20000 + len(header)) # the EAGAIN prevents it from reading the last 100 bytes c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() self.assertEquals(pos, 4096 + 4096) # now tell it to read the last 100 bytes c.handle_read(None, 0) c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() self.assertEquals(pos, 4096 + 4096 + 100)
def test_eagain_on_buffer_size(self): c = self.test_successful_connection() header = six.b('\x00\x00\x00\x00') + int32_pack(20000) responses = [ header + (six.b('a') * (4096 - len(header))), six.b('a') * 4096, socket_error(errno.EAGAIN), six.b('a') * 100, socket_error(errno.EAGAIN)] def side_effect(*args): response = responses.pop(0) if isinstance(response, socket_error): raise response else: return response self.get_socket(c).recv.side_effect = side_effect c.handle_read(*self.null_handle_function_args) self.assertEqual(c._current_frame.end_pos, 20000 + len(header)) # the EAGAIN prevents it from reading the last 100 bytes c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() self.assertEqual(pos, 4096 + 4096) # now tell it to read the last 100 bytes c.handle_read(*self.null_handle_function_args) c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() self.assertEqual(pos, 4096 + 4096 + 100)
def test_egain_on_buffer_size(self, *args): # get a connection that's already fully started c = self.test_successful_connection() header = six.b('\x00\x00\x00\x00') + int32_pack(20000) responses = [ header + (six.b('a') * (4096 - len(header))), six.b('a') * 4096, socket_error(errno.EAGAIN), six.b('a') * 100, socket_error(errno.EAGAIN)] def side_effect(*args): response = responses.pop(0) if isinstance(response, socket_error): raise response else: return response c._socket.recv.side_effect = side_effect c.handle_read(None, 0) self.assertEqual(c._total_reqd_bytes, 20000 + len(header)) # the EAGAIN prevents it from reading the last 100 bytes c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() self.assertEqual(pos, 4096 + 4096) # now tell it to read the last 100 bytes c.handle_read(None, 0) c._iobuf.seek(0, os.SEEK_END) pos = c._iobuf.tell() self.assertEqual(pos, 4096 + 4096 + 100)
def serialize_safe(cls, val, protocol_version): proto_version = max(3, protocol_version) buf = io.BytesIO() for i, (fieldname, subtype) in enumerate(zip(cls.fieldnames, cls.subtypes)): # first treat as a tuple, else by custom type try: item = val[i] except TypeError: item = getattr(val, fieldname) if item is not None: packed_item = subtype.to_binary(item, proto_version) buf.write(int32_pack(len(packed_item))) buf.write(packed_item) else: buf.write(int32_pack(-1)) return buf.getvalue()
def serialize_safe(cls, val, protocol_version): proto_version = max(3, protocol_version) buf = io.BytesIO() for item, subtype in zip(val, cls.subtypes): packed_item = subtype.to_binary(item, proto_version) buf.write(int32_pack(len(packed_item))) buf.write(packed_item) return buf.getvalue()
def serialize(dec): sign, digits, exponent = dec.as_tuple() unscaled = int(''.join([str(digit) for digit in digits])) if sign: unscaled *= -1 scale = int32_pack(-exponent) unscaled = varint_pack(unscaled) return scale + unscaled
def serialize(dec): try: sign, digits, exponent = dec.as_tuple() except AttributeError: raise TypeError("Non-Decimal type received for Decimal value") unscaled = int(''.join([str(digit) for digit in digits])) if sign: unscaled *= -1 scale = int32_pack(-exponent) unscaled = varint_pack(unscaled) return scale + unscaled
def serialize(dec, protocol_version): try: sign, digits, exponent = dec.as_tuple() except AttributeError: raise TypeError("Non-Decimal type received for Decimal value") unscaled = int(''.join([str(digit) for digit in digits])) if sign: unscaled *= -1 scale = int32_pack(-exponent) unscaled = varint_pack(unscaled) return scale + unscaled
def to_string(self, stream_id, compression=None): body = StringIO() self.send_body(body) body = body.getvalue() version = PROTOCOL_VERSION | HEADER_DIRECTION_FROM_CLIENT flags = 0 if compression is not None and len(body) > 0: body = compression(body) flags |= 0x01 if self.tracing: flags |= 0x02 msglen = int32_pack(len(body)) msg_parts = map(int8_pack, (version, flags, stream_id, self.opcode)) + [msglen, body] return ''.join(msg_parts)
def serialize(dec, protocol_version): try: sign, digits, exponent = dec.as_tuple() except AttributeError: try: sign, digits, exponent = Decimal(dec).as_tuple() except Exception: raise TypeError("Invalid type for Decimal value: %r", dec) unscaled = int("".join([str(digit) for digit in digits])) if sign: unscaled *= -1 scale = int32_pack(-exponent) unscaled = varint_pack(unscaled) return scale + unscaled
def serialize(dec, protocol_version): try: sign, digits, exponent = dec.as_tuple() except AttributeError: try: sign, digits, exponent = Decimal(dec).as_tuple() except Exception: raise TypeError("Invalid type for Decimal value: %r", dec) unscaled = int(''.join([str(digit) for digit in digits])) if sign: unscaled *= -1 scale = int32_pack(-exponent) unscaled = varint_pack(unscaled) return scale + unscaled
def test_negative_body_length(self, *args): c = self.make_connection() c._requests = Mock() c.defunct = Mock() # read in a SupportedMessage response header = self.make_header_prefix(SupportedMessage) message = header + int32_pack(-13) c._iobuf = BytesIO() c._iobuf.write(message) c.process_io_buffer() # make sure it errored correctly c.defunct.assert_called_once_with(ANY) args, kwargs = c.defunct.call_args self.assertIsInstance(args[0], ProtocolError)
def send(self, f, streamid, compression=None): body = StringIO() self.send_body(body) body = body.getvalue() version = PROTOCOL_VERSION | HEADER_DIRECTION_FROM_CLIENT flags = 0 if compression is not None and len(body) > 0: body = compression(body) flags |= 0x01 if self.tracing: flags |= 0x02 msglen = int32_pack(len(body)) header = ''.join(map(int8_pack, (version, flags, streamid, self.opcode))) \ + msglen f.write(header) if len(body) > 0: f.write(body)
def lz4_compress(byts): # write length in big-endian instead of little-endian return int32_pack(len(byts)) + lz4.compress(byts)[4:]
def write_int(f, i): f.write(int32_pack(i))
def ip_of_key(ks, key): hosts = self.session.cluster.metadata.get_replicas( ks, int32_pack(key)) host = hosts[0] return host.address
def serialize(byts, protocol_version): return int32_pack(byts)
def decompress(self, encoded_data, uncompressed_length): return self.decompressor(int32_pack(uncompressed_length) + encoded_data)