def test_indexed_header_field(self): """ The header field representation uses an indexed header field, from the static table. """ d = Decoder() header_set = [(':method', 'GET')] data = b'\x82' assert d.decode(data) == header_set assert list(d.header_table.dynamic_entries) == []
def test_literal_header_field_without_indexing(self): """ The header field representation uses an indexed name and a literal value. """ d = Decoder() header_set = [(':path', '/sample/path')] data = b'\x04\x0c/sample/path' assert d.decode(data) == header_set assert list(d.header_table.dynamic_entries) == []
def save(self): encoder = Encoder() payload = encoder.encode(self) self.length = len(payload) decoder = Decoder() check = decoder.decode(payload) logger.info('payload of the header is {}'.format(check)) base = super().save() return base + payload
def test_literal_never_indexed_emits_neverindexedheadertuple(self): """ A literal header field that must never be indexed emits a NeverIndexedHeaderTuple. """ d = Decoder() data = b'\x10\x0acustom-key\x0dcustom-header' headers = d.decode(data) assert len(headers) == 1 header = headers[0] assert isinstance(header, NeverIndexedHeaderTuple)
def test_indexed_never_indexed_emits_neverindexedheadertuple(self): """ A header field with an indexed name that must never be indexed emits a NeverIndexedHeaderTuple. """ d = Decoder() data = b'\x14\x0c/sample/path' headers = d.decode(data) assert len(headers) == 1 header = headers[0] assert isinstance(header, NeverIndexedHeaderTuple)
def test_literal_header_field_with_indexing_emits_headertuple(self): """ A header field with indexing emits a HeaderTuple. """ d = Decoder() data = b'\x00\x0acustom-key\x0dcustom-header' headers = d.decode(data) assert len(headers) == 1 header = headers[0] assert isinstance(header, HeaderTuple) assert not isinstance(header, NeverIndexedHeaderTuple)
def test_literal_header_field_with_indexing(self): """ The header field representation uses a literal name and a literal value. """ d = Decoder() header_set = [('custom-key', 'custom-header')] data = b'\x40\x0acustom-key\x0dcustom-header' assert d.decode(data) == header_set assert list(d.header_table.dynamic_entries) == [ (n.encode('utf-8'), v.encode('utf-8')) for n, v in header_set ]
def test_request_examples_with_huffman(self): """ This section shows the same examples as the previous section, but using Huffman encoding for the literal values. """ d = Decoder() first_header_set = [ (':method', 'GET',), (':scheme', 'http',), (':path', '/',), (':authority', 'www.example.com'), ] first_data = ( b'\x82\x86\x84\x01\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0\xab\x90\xf4\xff' ) assert d.decode(first_data) == first_header_set assert list(d.header_table.dynamic_entries) == [] second_header_set = [ (':method', 'GET',), (':scheme', 'http',), (':path', '/',), (':authority', 'www.example.com',), ('cache-control', 'no-cache'), ] second_data = ( b'\x82\x86\x84\x01\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0\xab\x90\xf4\xff' b'\x0f\t\x86\xa8\xeb\x10d\x9c\xbf' ) assert d.decode(second_data) == second_header_set assert list(d.header_table.dynamic_entries) == [] third_header_set = [ (':method', 'GET',), (':scheme', 'https',), (':path', '/index.html',), (':authority', 'www.example.com',), ('custom-key', 'custom-value'), ] third_data = ( b'\x82\x87\x85\x01\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0\xab\x90\xf4\xff@' b'\x88%\xa8I\xe9[\xa9}\x7f\x89%\xa8I\xe9[\xb8\xe8\xb4\xbf' ) assert d.decode(third_data) == third_header_set assert len(d.header_table.dynamic_entries) == 1
def test_request_examples_without_huffman(self): """ This section shows several consecutive header sets, corresponding to HTTP requests, on the same connection. """ d = Decoder() first_header_set = [ (':method', 'GET',), (':scheme', 'http',), (':path', '/',), (':authority', 'www.example.com'), ] # The first_header_table doesn't contain 'authority' first_data = b'\x82\x86\x84\x01\x0fwww.example.com' assert d.decode(first_data) == first_header_set assert list(d.header_table.dynamic_entries) == [] # This request takes advantage of the differential encoding of header # sets. second_header_set = [ (':method', 'GET',), (':scheme', 'http',), (':path', '/',), (':authority', 'www.example.com',), ('cache-control', 'no-cache'), ] second_data = ( b'\x82\x86\x84\x01\x0fwww.example.com\x0f\t\x08no-cache' ) assert d.decode(second_data) == second_header_set assert list(d.header_table.dynamic_entries) == [] third_header_set = [ (':method', 'GET',), (':scheme', 'https',), (':path', '/index.html',), (':authority', 'www.example.com',), ('custom-key', 'custom-value'), ] third_data = ( b'\x82\x87\x85\x01\x0fwww.example.com@\ncustom-key\x0ccustom-value' ) assert d.decode(third_data) == third_header_set # Don't check the header table here, it's just too complex to be # reliable. Check its length though. assert len(d.header_table.dynamic_entries) == 1
def test_can_decode_multiple_header_table_size_changes(self): """ If multiple header table size changes are sent in at once, they are successfully decoded. """ d = Decoder() data = b'?a?\xe1\x1f\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99' expect = [ (':method', 'GET'), (':scheme', 'https'), (':path', '/'), (':authority', '127.0.0.1:8443') ] assert d.decode(data) == expect
def test_raw_decoding(self): """ The header field representation is decoded as a raw byte string instead of UTF-8 """ d = Decoder() header_set = [ (b'\x00\x01\x99\x30\x11\x22\x55\x21\x89\x14', b'custom-header') ] data = ( b'\x40\x0a\x00\x01\x99\x30\x11\x22\x55\x21\x89\x14\x0d' b'custom-header' ) assert d.decode(data, raw=True) == header_set
def test_truncated_header_name(self): """ If a header name is truncated an error is raised. """ d = Decoder() # This is a simple header block that has a bad ending. The interesting # part begins on the second line. This indicates a string that has # literal name and value. The name is a 5 character huffman-encoded # string that is only three bytes long. data = ( b'\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99' b'\x00\x85\xf2\xb2J' ) with pytest.raises(HPACKDecodingError): d.decode(data)
def test_can_encode_a_story_with_huffman(self, raw_story): d = Decoder() e = Encoder() for case in raw_story['cases']: # The input headers are a list of dicts, which is annoying. input_headers = [ (item[0], item[1]) for header in case['headers'] for item in header.items() ] encoded = e.encode(input_headers, huffman=True) decoded_headers = d.decode(encoded) assert input_headers == decoded_headers
class HeadersFrame(Frame): type = 0x1 allowed_flags = [ Flag('END_STREAM', 0x1), Flag('END_HEADERS', 0x4), Flag('PADDED', 0x8), Flag('PRIORITY', 0x20) ] header_decoder = Decoder() def __init__(self, stream_id, data=b'', **kwargs): super().__init__(stream_id, **kwargs) self.data = data def parse_payload(self, payload): pos = 0 if 'PADDED' in self.flags: self.pad_length = payload[pos] pos += 1 else: self.pad_length = 0 if 'PRIORITY' in self.flags: self.stream_dependency = struct.unpack('!L', payload[pos:pos + 4])[0] self.exclusive = (self.stream_dependency >> 31) & 1 self.stream_dependency &= 0x7FFFFFFF pos += 4 self.weight = payload[pos] + 1 pos += 1 self.header_block = payload[pos:len(payload) - self.pad_length] self.headers = self.header_decoder.decode(self.header_block) def serialize_payload(self): padding_data = b'' padding = b'\0' * 0 if 'PRIORITY' in self.flags: priority_data = b'' else: priority_data = b'' return b''.join([padding_data, priority_data, self.data, padding])
class PushPromiseFrame(Frame): type = 0x5 allowed_flags = [Flag('END_HEADERS', 0x4), Flag('PADDED', 0x8)] header_decoder = Decoder() def parse_payload(self, payload): pos = 0 if 'PADDED' in self.flags: self.pad_length = payload[pos] pos += 1 else: self.pad_length = 0 self.promised_stream_id = struct.unpack('!I', payload[pos:pos + 4])[0] pos += 4 self.header_block = payload[pos:len(payload) - self.pad_length] self.headers = self.header_decoder.decode(self.header_block)
def test_truncated_header_value(self): """ If a header value is truncated an error is raised. """ d = Decoder() # This is a simple header block that has a bad ending. The interesting # part begins on the second line. This indicates a string that has # literal name and value. The name is a 5 character huffman-encoded # string, but the entire EOS character has been written over the end. # This causes hpack to see the header value as being supposed to be # 622462 bytes long, which it clearly is not, and so this must fail. data = ( b'\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99' b'\x00\x85\xf2\xb2J\x87\xff\xff\xff\xfd%B\x7f' ) with pytest.raises(HPACKDecodingError): d.decode(data)
def test_can_encode_a_story_no_huffman(self, raw_story): d = Decoder() e = Encoder() for case in raw_story['cases']: # The input headers are a list of dicts, which is annoying. input_headers = [ (item[0], item[1]) for header in case['headers'] for item in header.items() ] encoded = e.encode(input_headers, huffman=False) decoded_headers = d.decode(encoded) assert input_headers == decoded_headers assert all( isinstance(header, HeaderTuple) for header in decoded_headers )
def test_resizing_header_table(self): # We need to decode a substantial number of headers, to populate the # header table. This string isn't magic: it's the output from the # equivalent test for the Encoder. d = Decoder() data = ( b'\x82\x87D\x87a\x07\xa4\xacV4\xcfA\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0' b'\xab\x90\xf4\xff@\x88%\xa8I\xe9[\xa9}\x7f\x89%\xa8I\xe9[\xb8\xe8' b'\xb4\xbfz\xbc\xd0\x7ff\xa2\x81\xb0\xda\xe0S\xfa\xd02\x1a\xa4\x9d' b'\x13\xfd\xa9\x92\xa4\x96\x854\x0c\x8aj\xdc\xa7\xe2\x81\x02\xef}' b'\xa9g{\x81qp\x7fjb):\x9d\x81\x00 \x00@\x150\x9a\xc2\xca\x7f,\x05' b'\xc5\xc1S\xb0I|\xa5\x89\xd3M\x1fC\xae\xba\x0cA\xa4\xc7\xa9\x8f3' b'\xa6\x9a?\xdf\x9ah\xfa\x1du\xd0b\r&=Ly\xa6\x8f\xbe\xd0\x01w\xfe' b'\xbeX\xf9\xfb\xed\x00\x17{@\x8a\xfc[=\xbdF\x81\xad\xbc\xa8O\x84y' b'\xe7\xde\x7f' ) d.decode(data) # Resize the header table to a size so small that nothing can be in it. d.header_table_size = 40 assert len(d.header_table.dynamic_entries) == 0
def defuck(data): next_f = 0 errn = 0 while len(data) > next_f + 9: if errn > 2: break try: nframe, _len = Frame.parse_frame_header(data[next_f:next_f + 9]) nframe.parse_body(memoryview(data[next_f + 9:next_f + 9 + _len])) print(nframe) for i in nframe.__dict__: if i == 'data': data = nframe.data dd = Decoder() print('frame->data->decode : ', dd.decode(data)) next_f += _len + 9 except Exception as e: print(e) errn += 1 next_f += _len + 9 continue
def update_http2(self, packet): ''' Decodes the http2 header and updates the AppTracker with the relevant information (domain name, cookies, post data, custom headers, and uri query parameters) from the http2 packet capture. Ignores any packet that fails to decode. @params: packet is a http2 header packet. ''' try: d = Decoder() decoded_headers = d.decode( bytes.fromhex(packet.http2.headers.raw_value)) #print(decoded_headers) for (key, value) in decoded_headers: if key == ":authority": self.domain = value elif key == ":path": idx = value.find('?') if idx != -1: params = value[idx + 1:] if params not in self.uris: self.uris.append(params) elif key == "cookie": if value not in self.cookies: self.cookies.append(value) elif key.lower() not in http_headers \ and key.lower() not in http2_headers: h = key.lower() + ": " + value if h not in self.headers: self.headers.append(h) elif key == "user_agent": print(value) elif key == ":scheme" and value == "https": self.isSSL = True except Exception as e: #print(e) pass
def test_apache_trafficserver(self): # This test reproduces the bug in #110, using exactly the same header # data. d = Decoder() data = ( b'\x10\x07:status\x03200@\x06server\tATS/6.0.0' b'@\x04date\x1dTue, 31 Mar 2015 08:09:51 GMT' b'@\x0ccontent-type\ttext/html@\x0econtent-length\x0542468' b'@\rlast-modified\x1dTue, 31 Mar 2015 01:55:51 GMT' b'@\x04vary\x0fAccept-Encoding@\x04etag\x0f"5519fea7-a5e4"' b'@\x08x-served\x05Nginx@\x14x-subdomain-tryfiles\x04True' b'@\x07x-deity\thydra-lts@\raccept-ranges\x05bytes@\x03age\x010' b'@\x19strict-transport-security\rmax-age=86400' b'@\x03via2https/1.1 ATS (ApacheTrafficServer/6.0.0 [cSsNfU])' ) expect = [ (':status', '200'), ('server', 'ATS/6.0.0'), ('date', 'Tue, 31 Mar 2015 08:09:51 GMT'), ('content-type', 'text/html'), ('content-length', '42468'), ('last-modified', 'Tue, 31 Mar 2015 01:55:51 GMT'), ('vary', 'Accept-Encoding'), ('etag', '"5519fea7-a5e4"'), ('x-served', 'Nginx'), ('x-subdomain-tryfiles', 'True'), ('x-deity', 'hydra-lts'), ('accept-ranges', 'bytes'), ('age', '0'), ('strict-transport-security', 'max-age=86400'), ('via', 'https/1.1 ATS (ApacheTrafficServer/6.0.0 [cSsNfU])'), ] result = d.decode(data) assert result == expect # The status header shouldn't be indexed. assert len(d.header_table.dynamic_entries) == len(expect) - 1
def genFrame(data): next_f = 0 errn = 0 while len(data) > next_f+9: print("[*] "+"-"*30) if errn > 2: break try: nframe, _len = Frame.parse_frame_header( data[next_f:next_f+9]) nframe.parse_body(memoryview(data[next_f+9:next_f+9 + _len])) print("[+] Frame -> %s" % nframe) for i in nframe.__dict__: if i == 'data': print("[+] Data:") print("[+] ", Decoder().decode(nframe.data)) print("[+] ") next_f += _len + 9 except Exception as e: print(e) errn += 1 next_f += _len + 9 continue
def __init__(self, length: int, type_, flags: bytes, stream_identifier: int, data=None): super(Headers, self).__init__(length, type_, flags, stream_identifier) logger.debug('Headers is called.') self.end_stream = HeadersFlags.END_STREAM.value & self.flags self.end_headers = HeadersFlags.END_HEADERS.value & self.flags self.padded = HeadersFlags.PADDED.value & self.flags self.priority = HeadersFlags.PRIORITY.value & self.flags logger.debug('{}, {}, {}, {}'.format(self.end_stream, self.end_headers, self.padded, self.priority)) payload = BytesIO(data) if self.padded: payload.read(1) if self.priority: # TODO: handle priority properly self.stream_dependency = int.from_bytes(payload.read(4), 'big', signed=False) self.priority_weight = int.from_bytes(payload.read(1), 'big', signed=False) logger.debug('stream_dependency: {}, '.format(self.stream_dependency) +\ 'priority_weight: {}'.format(self.priority_weight)) if length: decoder = Decoder() fields = decoder.decode(payload.read()) for k, v in fields: self[k] = v logger.debug('{}: {}'.format(k, v))
def test_ordering_applies_to_encoding(self, special_keys, boring_keys): """ When encoding a dictionary the special keys all appear first. """ def _prepend_colon(k): if isinstance(k, str): return ':' + k else: return b':' + k special_keys = set(map(_prepend_colon, special_keys)) input_dict = { k: b'testval' for k in itertools.chain( special_keys, boring_keys ) } e = Encoder() d = Decoder() encoded = e.encode(input_dict) decoded = iter(d.decode(encoded, raw=True)) received_special = set() received_boring = set() expected_special = set(map(_to_bytes, special_keys)) expected_boring = set(map(_to_bytes, boring_keys)) for _ in special_keys: k, _ = next(decoded) received_special.add(k) for _ in boring_keys: k, _ = next(decoded) received_boring.add(k) assert expected_special == received_special assert expected_boring == received_boring
def __init__(self, logger, ip_manager, config, ssl_sock, close_cb, retry_task_cb, idle_cb, log_debug_data): super(Http2Worker, self).__init__(logger, ip_manager, config, ssl_sock, close_cb, retry_task_cb, idle_cb, log_debug_data) self.network_buffer_size = 65535 # Google http/2 time out is 4 mins. self.ssl_sock.settimeout(240) self._sock = BufferedSocket(ssl_sock, self.network_buffer_size) self.next_stream_id = 1 self.streams = {} self.last_ping_time = time.time() self.continue_timeout = 0 # count ping not ACK # increase when send ping # decrease when recv ping ack # if this in not 0, don't accept request. self.ping_on_way = 0 self.accept_task = False # request_lock self.request_lock = threading.Lock() # all send frame must put to this queue # then send by send_loop # every frame put to this queue must allowed by stream window and connection window # any data frame blocked by connection window should put to self.blocked_send_frames self.send_queue = Queue.Queue() self.encoder = Encoder() self.decoder = Decoder() # keep blocked data frame in this buffer # which is allowed by stream window but blocked by connection window. # They will be sent when connection window open self.blocked_send_frames = [] # Values for the settings used on an HTTP/2 connection. # will send to remote using Setting Frame self.local_settings = { SettingsFrame.INITIAL_WINDOW_SIZE: 16 * 1024 * 1024, SettingsFrame.SETTINGS_MAX_FRAME_SIZE: 256 * 1024 } self.local_connection_initial_windows = 32 * 1024 * 1024 self.local_window_manager = FlowControlManager( self.local_connection_initial_windows) # changed by server, with SettingFrame self.remote_settings = { SettingsFrame.INITIAL_WINDOW_SIZE: DEFAULT_WINDOW_SIZE, SettingsFrame.SETTINGS_MAX_FRAME_SIZE: DEFAULT_MAX_FRAME, SettingsFrame.MAX_CONCURRENT_STREAMS: 100 } #self.remote_window_size = DEFAULT_WINDOW_SIZE self.remote_window_size = 32 * 1024 * 1024 # send Setting frame before accept task. self._send_preamble() threading.Thread(target=self.send_loop).start() threading.Thread(target=self.recv_loop).start()
def __init__(self): super().__init__() self.decoder = Decoder() self._factory = {klass.FrameType(): klass for klass in FrameBase.__subclasses__()}
from hpack import Decoder d = Decoder() decoded = d.decode( bytes.fromhex( "1de514084569a62a81716c49c2166e337bd46d4e501d0ef8037795422ec8d406e9d600c547355a6b69f28d8fc4ef4556f8351f8e09ae116b7a20c25943173baa216fa4aeb3d9d9b278cfb46bffd4310516d226c30f31f4569938826142517c2eedb08c4ca1cd14e3abe89565be71492955eed9eb25f7109069e6d5b86fd5974976b290b635be9054fda871c679a93ec18454f4dc9703bd7e1a0c7b8ed0d50c6644b96461b07c3640d99bb04986040b385dce2f2172c3a0f364d99d8fdaefc8ba0113f9b4942cbc61d8d560b8b7b7918fddea11362d390321c442ba8c2ea5352c79b3adea93f5050a78cd20508b4e69241467e3c3adc6e40d9244e02dc9ea60b6fa820563e196f03b2f85fb7e856fd90daea44d9e3bb8a4ef9eec9182eb72a1dde0fadaba9809139d366777436fb058d4e7694ea409eb169c0e9bc649d677a08f0535fcec78285d84fdf9762433af71cf53dafa90618d7f765ac3d75b96002893ea89ebefdf7c1411d0491f6d718c6e5f12b9262d61ec141817bd97d41d58817863ac58c839919ba889947c78db66346d95279c6a30da2b2b25dded0e27d77a7d0d71ea0bbdf3d24e33f9a7ee475a1c3bd62094603c6c151e788db3732c4efdecd5197afd448b7dd025130f483f72e99834b6da2cb23368401efe4186270d97f76394fd8d6207d2509b70b8c83005824686172ba8fd9c62a7cd8cb02394af97855d437a9f20da2757e9f61a68601cbda3405bb730135192d93d3b6a04e14a36f8d5a1e7ac9d8b90ba366b0570753c07fed2d1820cab015a12bc330444aa7a60c6966d2ac188a13e7048b2e3e218fdbc02c7481f64af2f1d2bf77d4adf94820726a22cf91a6f138c2ae21b60d680859175da36f3e625dc5b4da77fa22ff1de08a0ace0bde0b0148e4fe886f4a13e9dc0c95fb2005bfa3372ea738b08ca267f6f5e256189930e249659a7e9e329374999ed96bf3885325acf2744416d37090a4c10946b6dc897c97731537f14072f3b2f2fa97b315c24217aac426ecc32226ad1842b29e7abba4624de84b60eb331c21a1465c92df52f8770a310ee21261b7b8b3a15e1b5ee243fa81968d7ee76c032dd7e5a07a4e0858a0f22f24c2e004c1b447935f8bfa8852c960e434349fed3e0193429678345b52631143ae2b57ecee2eb154b93b32d6773deb80acb0d768c255a3e69ba7d4845b8542a5841b56decbb108032f233369a46709804aa79f76dd61f60715cd1292997fba8c3fd66329b8106c75649fa6e08e30d2629fc6a7b6a066a8f52bfa78272e4bec751d726026f050555837f4082d8beb157a4ab4509507518b051d207ceb2edbad6dca8a1fb28be69239ca28d62a6a197ef865c01a5a1768e43026b3faf8ba4bf8a24d26decb9e16db252c797578f60cfc21fe4f44b99e88ba893d596bdfcc88c7e9d9bb4bbecf78d76fe3408377805a0467a407cbdd22d03cad9dcedc7c21731242e86883a7fcce1e1e225bf7470036558715a55fed1579d2d0a" )) print(decoded)
import logging from logging.config import dictConfig import pyshark from h2.config import H2Configuration from h2.connection import H2Connection from h2.frame_buffer import FrameBuffer from h2.frame_buffer import HeadersFrame from hyperframe.frame import * from hpack import Decoder log_conf = json.load(open("logging.json")) dictConfig(log_conf) logger = logging.getLogger() # HTTP2 Header decoder decoder = Decoder() # Change INPUT_CONFIG to your pcap file's name and server/client's IP address INPUT_CONFIG = { "pcap": "./pcap-sample/z.pcap", "client": "10.20.50.25", "server": "10.20.50.34", } pcap_file = INPUT_CONFIG["pcap"] client_ip = INPUT_CONFIG["client"] server_ip = INPUT_CONFIG["server"] logger.debug("Input config: {0}".format(INPUT_CONFIG)) # Change STREAM_NUMBER to the stream number you want to follow. # STREAM_NUMBER = 0 # Switching protocol check
def test_header_table_size_getter(self): d = Decoder() assert d.header_table_size