def read_header_format(self, sz, header_size, data): # clear out any previous transforms self.__read_transforms = [] header_size = header_size * 4 if header_size > sz: raise TTransportException(TTransportException.INVALID_FRAME_SIZE, "Header size is larger than frame") end_header = header_size + data.tell() self.__proto_id = readVarint(data) num_headers = readVarint(data) if self.__proto_id == 1 and self.__client_type != \ CLIENT_TYPE.HTTP_SERVER: raise TTransportException(TTransportException.INVALID_CLIENT_TYPE, "Trying to recv JSON encoding over binary") # Read the headers. Data for each header varies. for h in range(0, num_headers): trans_id = readVarint(data) if trans_id == TRANSFORM.ZLIB: self.__read_transforms.insert(0, trans_id) elif trans_id == TRANSFORM.SNAPPY: self.__read_transforms.insert(0, trans_id) elif trans_id == TRANSFORM.HMAC: raise TApplicationException( TApplicationException.INVALID_TRANSFORM, "Hmac transform is no longer supported: %i" % trans_id) else: # TApplicationException will be sent back to client raise TApplicationException( TApplicationException.INVALID_TRANSFORM, "Unknown transform in client request: %i" % trans_id) # Clear out previous info headers. self.__read_headers.clear() # Read the info headers. while data.tell() < end_header: info_id = readVarint(data) if info_id == INFO.NORMAL: _read_info_headers( data, end_header, self.__read_headers) elif info_id == INFO.PERSISTENT: _read_info_headers( data, end_header, self.__read_persistent_headers) else: break # Unknown header. Stop info processing. if self.__read_persistent_headers: self.__read_headers.update(self.__read_persistent_headers) # Skip the rest of the header data.seek(end_header) payload = data.read(sz - header_size) # Read the data section. self.__rbuf = StringIO(self.untransform(payload))
def read_header_format(self, sz, header_size, data): # clear out any previous transforms self.__read_transforms = [] header_size = header_size * 4 if header_size > sz: raise TTransportException(TTransportException.INVALID_FRAME_SIZE, "Header size is larger than frame") end_header = header_size + data.tell() self.__proto_id = readVarint(data) num_headers = readVarint(data) if self.__proto_id == 1 and self.__client_type != \ CLIENT_TYPE.HTTP_SERVER: raise TTransportException(TTransportException.INVALID_CLIENT_TYPE, "Trying to recv JSON encoding over binary") # Read the headers. Data for each header varies. for _ in range(0, num_headers): trans_id = readVarint(data) if trans_id in (TRANSFORM.ZLIB, TRANSFORM.SNAPPY, TRANSFORM.ZSTD): self.__read_transforms.insert(0, trans_id) elif trans_id == TRANSFORM.HMAC: raise TApplicationException( TApplicationException.INVALID_TRANSFORM, "Hmac transform is no longer supported: %i" % trans_id) else: # TApplicationException will be sent back to client raise TApplicationException( TApplicationException.INVALID_TRANSFORM, "Unknown transform in client request: %i" % trans_id) # Clear out previous info headers. self.__read_headers.clear() # Read the info headers. while data.tell() < end_header: info_id = readVarint(data) if info_id == INFO.NORMAL: _read_info_headers( data, end_header, self.__read_headers) elif info_id == INFO.PERSISTENT: _read_info_headers( data, end_header, self.__read_persistent_headers) else: break # Unknown header. Stop info processing. if self.__read_persistent_headers: self.__read_headers.update(self.__read_persistent_headers) # Skip the rest of the header data.seek(end_header) payload = data.read(sz - header_size) # Read the data section. self.__rbuf = StringIO(self.untransform(payload))
def _parse_header_format(self, buffer): # make BufferIO look like TTransport for varint helpers buffer_transport = TMemoryBuffer() buffer_transport._buffer = buffer buffer.read(2) # discard the magic bytes self.flags, = U16.unpack(buffer.read(U16.size)) self.sequence_id, = I32.unpack(buffer.read(I32.size)) header_length = U16.unpack(buffer.read(U16.size))[0] * 4 end_of_headers = buffer.tell() + header_length if end_of_headers > len(buffer.getvalue()): raise TTransportException( TTransportException.SIZE_LIMIT, "Header size is larger than whole frame.", ) self._protocol_id = readVarint(buffer_transport) transforms = [] transform_count = readVarint(buffer_transport) for _ in range(transform_count): transform_id = readVarint(buffer_transport) if transform_id not in READ_TRANSFORMS_BY_ID: raise TApplicationException( TApplicationException.INVALID_TRANSFORM, "Unknown transform: %d" % transform_id, ) transforms.append(transform_id) transforms.reverse() headers = {} while buffer.tell() < end_of_headers: header_type = readVarint(buffer_transport) if header_type == TInfoHeaderType.KEY_VALUE: count = readVarint(buffer_transport) for _ in range(count): key = _readString(buffer_transport) value = _readString(buffer_transport) headers[key] = value else: break # ignore unknown headers self._read_headers = headers # skip padding / anything we didn't understand buffer.seek(end_of_headers) payload = buffer.read() for transform_id in transforms: transform_fn = READ_TRANSFORMS_BY_ID[transform_id] payload = transform_fn(payload) return BufferIO(payload)
def _read_string(bufio, buflimit): str_sz = readVarint(bufio) if str_sz + bufio.tell() > buflimit: raise TTransportException( TTransportException.INVALID_FRAME_SIZE, "String read too big" ) return bufio.read(str_sz)
def _readString(trans): size = readVarint(trans) if size < 0: raise TTransportException( TTransportException.NEGATIVE_SIZE, "Negative length" ) return trans.read(size)
def _read_info_headers(data, end_header, read_headers): num_keys = readVarint(data) for _ in xrange(num_keys): str_key = _read_string(data, end_header) str_value = _read_string(data, end_header) read_headers[str_key] = str_value
def read_header_format(self, sz, header_size, data): # clear out any previous transforms self.__read_transforms = [] header_size = header_size * 4 if header_size > sz: raise TTransportException(TTransportException.INVALID_FRAME_SIZE, "Header size is larger than frame") end_header = header_size + data.tell() self.__proto_id = readVarint(data) num_headers = readVarint(data) if self.__proto_id == 1 and self.__client_type != \ self.HTTP_CLIENT_TYPE: raise TTransportException( TTransportException.INVALID_CLIENT_TYPE, "Trying to recv JSON encoding over binary") # Read the headers. Data for each header varies. hmac_sz = 0 for h in range(0, num_headers): trans_id = readVarint(data) if trans_id == self.ZLIB_TRANSFORM: self.__read_transforms.insert(0, trans_id) elif trans_id == self.SNAPPY_TRANSFORM: self.__read_transforms.insert(0, trans_id) elif trans_id == self.HMAC_TRANSFORM: hmac_sz = ord(data.read(1)) data.seek(-1, os.SEEK_CUR) data.write(b'\0') else: # TApplicationException will be sent back to client raise TApplicationException( TApplicationException.INVALID_TRANSFORM, "Unknown transform in client request: %i" % trans_id) # Clear out previous info headers. self.__read_headers.clear() # Read the info headers. while data.tell() < end_header: info_id = readVarint(data) if info_id == self.INFO_KEYVALUE: THeaderTransport._read_info_headers(data, end_header, self.__read_headers) elif info_id == self.INFO_PKEYVALUE: THeaderTransport._read_info_headers( data, end_header, self.__read_persistent_headers) else: break # Unknown header. Stop info processing. if self.__read_persistent_headers: self.__read_headers.update(self.__read_persistent_headers) # Skip the rest of the header data.seek(end_header) payload = data.read(sz - header_size - hmac_sz) # Verify the mac if self.__hmac_verify_func: hmac = data.read(hmac_sz) verify_data = data.getvalue()[:-hmac_sz] if not self.__hmac_verify_func(verify_data, hmac): raise TTransportException( TTransportException.INVALID_TRANSFORM, "HMAC did not verify") # Read the data section. self.__rbuf = StringIO(self.untransform(payload))
def _read_info_headers(data, end_header, read_headers): num_keys = readVarint(data) for _ in xrange(num_keys): str_key = THeaderTransport._read_string(data, end_header) str_value = THeaderTransport._read_string(data, end_header) read_headers[str_key] = str_value
def _read_string(bufio, buflimit): str_sz = readVarint(bufio) if str_sz + bufio.tell() > buflimit: raise TTransportException(TTransportException.INVALID_FRAME_SIZE, "String read too big") return bufio.read(str_sz)
def read_header_format(self, sz, header_size, data): # clear out any previous transforms self.__read_transforms = [] header_size = header_size * 4 if header_size > sz: raise TTransportException(TTransportException.INVALID_FRAME_SIZE, "Header size is larger than frame") end_header = header_size + data.tell() self.__proto_id = readVarint(data) num_headers = readVarint(data) if self.__proto_id == 1 and self.__client_type != \ self.HTTP_CLIENT_TYPE: raise TTransportException(TTransportException.INVALID_CLIENT_TYPE, "Trying to recv JSON encoding over binary") # Read the headers. Data for each header varies. hmac_sz = 0 for h in range(0, num_headers): trans_id = readVarint(data) if trans_id == self.ZLIB_TRANSFORM: self.__read_transforms.insert(0, trans_id) elif trans_id == self.SNAPPY_TRANSFORM: self.__read_transforms.insert(0, trans_id) elif trans_id == self.HMAC_TRANSFORM: hmac_sz = ord(data.read(1)) data.seek(-1, os.SEEK_CUR) data.write(b'\0') else: # TApplicationException will be sent back to client raise TApplicationException( TApplicationException.INVALID_TRANSFORM, "Unknown transform in client request: %i" % trans_id) # Clear out previous info headers. self.__read_headers.clear() # Read the info headers. while data.tell() < end_header: info_id = readVarint(data) if info_id == self.INFO_KEYVALUE: THeaderTransport._read_info_headers(data, end_header, self.__read_headers) elif info_id == self.INFO_PKEYVALUE: THeaderTransport._read_info_headers(data, end_header, self.__read_persistent_headers) else: break # Unknown header. Stop info processing. if self.__read_persistent_headers: self.__read_headers.update(self.__read_persistent_headers) # Skip the rest of the header data.seek(end_header) payload = data.read(sz - header_size - hmac_sz) # Verify the mac if self.__hmac_verify_func: hmac = data.read(hmac_sz) verify_data = data.getvalue()[:-hmac_sz] if not self.__hmac_verify_func(verify_data, hmac): raise TTransportException(TTransportException.INVALID_TRANSFORM, "HMAC did not verify") # Read the data section. self.__rbuf = StringIO(self.untransform(payload))