def test_invalid_indexed_literal(self):
d = Decoder()
# Refer to an index that is too large.
data = b'\x82\x86\x84\x7f\x0a\x0fwww.example.com'
with pytest.raises(InvalidTableIndex):
d.decode(data)
def test_invalid_indexed_header(self):
d = Decoder()
# Refer to an indexed header that is too large.
data = b'\xBE\x86\x84\x01\x0fwww.example.com'
with pytest.raises(InvalidTableIndex):
d.decode(data)
def test_utf8_errors_raise_hpack_decoding_error(self):
d = Decoder()
# Invalid UTF-8 data.
data = b'\x82\x86\x84\x01\x10www.\x07\xaa\xd7\x95\xd7\xa8\xd7\x94.com'
with pytest.raises(HPACKDecodingError):
d.decode(data)
def test_max_header_list_size(self):
"""
If the header block is larger than the max_header_list_size, the HPACK
decoder throws an OversizedHeaderListError.
"""
d = Decoder(max_header_list_size=44)
data = b'\x14\x0c/sample/path'
with pytest.raises(OversizedHeaderListError):
d.decode(data)
def test_table_size_middle_rejected(self):
"""
If a header table size change comes anywhere but first in the header
block, it is forbidden.
"""
d = Decoder()
data = b'\x82?a\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99'
with pytest.raises(HPACKDecodingError):
d.decode(data)
def test_header_table_size_change_above_maximum(self):
"""
If a header table size change is received that exceeds the maximum
allowed table size, it is rejected.
"""
d = Decoder()
d.max_allowed_table_size = 127
data = b'?a\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99'
with pytest.raises(InvalidTableSizeError):
d.decode(data)
def test_table_size_not_adjusting(self):
"""
If the header table size is shrunk, and then the remote peer doesn't
join in the shrinking, then an error is raised.
"""
d = Decoder()
d.max_allowed_table_size = 128
data = b'\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99'
with pytest.raises(InvalidTableSizeError):
d.decode(data)
def test_request_examples_with_huffman(self):
"""
This section shows the same examples as the previous section, but
using Huffman encoding for the literal values.
"""
d = Decoder()
first_header_set = [
(':method', 'GET',),
(':scheme', 'http',),
(':path', '/',),
(':authority', 'www.example.com'),
]
first_data = (
b'\x82\x86\x84\x01\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0\xab\x90\xf4\xff'
)
assert d.decode(first_data) == first_header_set
assert list(d.header_table.dynamic_entries) == []
second_header_set = [
(':method', 'GET',),
(':scheme', 'http',),
(':path', '/',),
(':authority', 'www.example.com',),
('cache-control', 'no-cache'),
]
second_data = (
b'\x82\x86\x84\x01\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0\xab\x90\xf4\xff'
b'\x0f\t\x86\xa8\xeb\x10d\x9c\xbf'
)
assert d.decode(second_data) == second_header_set
assert list(d.header_table.dynamic_entries) == []
third_header_set = [
(':method', 'GET',),
(':scheme', 'https',),
(':path', '/index.html',),
(':authority', 'www.example.com',),
('custom-key', 'custom-value'),
]
third_data = (
b'\x82\x87\x85\x01\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0\xab\x90\xf4\xff@'
b'\x88%\xa8I\xe9[\xa9}\x7f\x89%\xa8I\xe9[\xb8\xe8\xb4\xbf'
)
assert d.decode(third_data) == third_header_set
assert len(d.header_table.dynamic_entries) == 1
def test_request_examples_without_huffman(self):
"""
This section shows several consecutive header sets, corresponding to
HTTP requests, on the same connection.
"""
d = Decoder()
first_header_set = [
(':method', 'GET',),
(':scheme', 'http',),
(':path', '/',),
(':authority', 'www.example.com'),
]
# The first_header_table doesn't contain 'authority'
first_data = b'\x82\x86\x84\x01\x0fwww.example.com'
assert d.decode(first_data) == first_header_set
assert list(d.header_table.dynamic_entries) == []
# This request takes advantage of the differential encoding of header
# sets.
second_header_set = [
(':method', 'GET',),
(':scheme', 'http',),
(':path', '/',),
(':authority', 'www.example.com',),
('cache-control', 'no-cache'),
]
second_data = (
b'\x82\x86\x84\x01\x0fwww.example.com\x0f\t\x08no-cache'
)
assert d.decode(second_data) == second_header_set
assert list(d.header_table.dynamic_entries) == []
third_header_set = [
(':method', 'GET',),
(':scheme', 'https',),
(':path', '/index.html',),
(':authority', 'www.example.com',),
('custom-key', 'custom-value'),
]
third_data = (
b'\x82\x87\x85\x01\x0fwww.example.com@\ncustom-key\x0ccustom-value'
)
assert d.decode(third_data) == third_header_set
# Don't check the header table here, it's just too complex to be
# reliable. Check its length though.
assert len(d.header_table.dynamic_entries) == 1
def test_truncated_header_name(self):
"""
If a header name is truncated an error is raised.
"""
d = Decoder()
# This is a simple header block that has a bad ending. The interesting
# part begins on the second line. This indicates a string that has
# literal name and value. The name is a 5 character huffman-encoded
# string that is only three bytes long.
data = (
b'\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99'
b'\x00\x85\xf2\xb2J'
)
with pytest.raises(HPACKDecodingError):
d.decode(data)
def parse_header(self,data):
d = Decoder()
decoded_headers = d.decode(data)
dic_header = {}
for str in decoded_headers:
dic_header[str[0]] = str[1]
return dic_header
def test_truncated_header_value(self):
"""
If a header value is truncated an error is raised.
"""
d = Decoder()
# This is a simple header block that has a bad ending. The interesting
# part begins on the second line. This indicates a string that has
# literal name and value. The name is a 5 character huffman-encoded
# string, but the entire EOS character has been written over the end.
# This causes hpack to see the header value as being supposed to be
# 622462 bytes long, which it clearly is not, and so this must fail.
data = (
b'\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99'
b'\x00\x85\xf2\xb2J\x87\xff\xff\xff\xfd%B\x7f'
)
with pytest.raises(HPACKDecodingError):
d.decode(data)
def test_literal_header_field_without_indexing(self):
"""
The header field representation uses an indexed name and a literal
value.
"""
d = Decoder()
header_set = [(':path', '/sample/path')]
data = b'\x04\x0c/sample/path'
assert d.decode(data) == header_set
assert list(d.header_table.dynamic_entries) == []
def test_indexed_header_field(self):
"""
The header field representation uses an indexed header field, from
the static table.
"""
d = Decoder()
header_set = [(':method', 'GET')]
data = b'\x82'
assert d.decode(data) == header_set
assert list(d.header_table.dynamic_entries) == []
def save(self):
encoder = Encoder()
payload = encoder.encode(self)
self.length = len(payload)
decoder = Decoder()
check = decoder.decode(payload)
logger.info('payload of the header is {}'.format(check))
base = super().save()
return base + payload
def test_resizing_header_table(self):
# We need to decode a substantial number of headers, to populate the
# header table. This string isn't magic: it's the output from the
# equivalent test for the Encoder.
d = Decoder()
data = (
b'\x82\x87D\x87a\x07\xa4\xacV4\xcfA\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0'
b'\xab\x90\xf4\xff@\x88%\xa8I\xe9[\xa9}\x7f\x89%\xa8I\xe9[\xb8\xe8'
b'\xb4\xbfz\xbc\xd0\x7ff\xa2\x81\xb0\xda\xe0S\xfa\xd02\x1a\xa4\x9d'
b'\x13\xfd\xa9\x92\xa4\x96\x854\x0c\x8aj\xdc\xa7\xe2\x81\x02\xef}'
b'\xa9g{\x81qp\x7fjb):\x9d\x81\x00 \x00@\x150\x9a\xc2\xca\x7f,\x05'
b'\xc5\xc1S\xb0I|\xa5\x89\xd3M\x1fC\xae\xba\x0cA\xa4\xc7\xa9\x8f3'
b'\xa6\x9a?\xdf\x9ah\xfa\x1du\xd0b\r&=Ly\xa6\x8f\xbe\xd0\x01w\xfe'
b'\xbeX\xf9\xfb\xed\x00\x17{@\x8a\xfc[=\xbdF\x81\xad\xbc\xa8O\x84y'
b'\xe7\xde\x7f'
)
d.decode(data)
# Resize the header table to a size so small that nothing can be in it.
d.header_table_size = 40
assert len(d.header_table.dynamic_entries) == 0
def test_indexed_never_indexed_emits_neverindexedheadertuple(self):
"""
A header field with an indexed name that must never be indexed emits a
NeverIndexedHeaderTuple.
"""
d = Decoder()
data = b'\x14\x0c/sample/path'
headers = d.decode(data)
assert len(headers) == 1
header = headers[0]
assert isinstance(header, NeverIndexedHeaderTuple)
def test_literal_header_field_with_indexing(self):
"""
The header field representation uses a literal name and a literal
value.
"""
d = Decoder()
header_set = [('custom-key', 'custom-header')]
data = b'\x40\x0acustom-key\x0dcustom-header'
assert d.decode(data) == header_set
assert list(d.header_table.dynamic_entries) == [
(n.encode('utf-8'), v.encode('utf-8')) for n, v in header_set
]
def test_literal_never_indexed_emits_neverindexedheadertuple(self):
"""
A literal header field that must never be indexed emits a
NeverIndexedHeaderTuple.
"""
d = Decoder()
data = b'\x10\x0acustom-key\x0dcustom-header'
headers = d.decode(data)
assert len(headers) == 1
header = headers[0]
assert isinstance(header, NeverIndexedHeaderTuple)
def test_literal_header_field_with_indexing_emits_headertuple(self):
"""
A header field with indexing emits a HeaderTuple.
"""
d = Decoder()
data = b'\x00\x0acustom-key\x0dcustom-header'
headers = d.decode(data)
assert len(headers) == 1
header = headers[0]
assert isinstance(header, HeaderTuple)
assert not isinstance(header, NeverIndexedHeaderTuple)
class Demo(object):
def __init__(self):
self.encoder = Encoder()
self.decoder = Decoder()
def run(self, headers):
origin_len = 0
encoded_len = 0
print "=" * 16
for header in headers:
header_tuple = header_dict_to_tuple(header)
encoded = self.encoder.encode([header_tuple])
encoded_len += len(encoded)
origin_len += len(header_tuple[0]) + len(header_tuple[1])
match = self.decoder.header_table.search(header_tuple[0], header_tuple[1])
print "{0}=>{1}".format(header, binascii.hexlify(encoded), translate_match(match))
print translate_match(match)
curr_state = None
length = 0
for b in encoded:
one_byte_data = bin(struct.unpack("B", b)[0])[2:].zfill(8)
curr_state, content, length = translate_byte(one_byte_data, match, curr_state, length)
if content:
print "{0} ({1})".format(one_byte_data, content)
self.decoder.decode(encoded)
print
print "Decompressed from {0} to {1}".format(origin_len, encoded_len)
print "=" * 16
def pretty_print_table(self, table):
for (k, v) in table.dynamic_entries:
print "{0}=>{1}".format(k, v)
def tables(self):
self.pretty_print_table(self.encoder.header_table)
def test_raw_decoding(self):
"""
The header field representation is decoded as a raw byte string instead
of UTF-8
"""
d = Decoder()
header_set = [
(b'\x00\x01\x99\x30\x11\x22\x55\x21\x89\x14', b'custom-header')
]
data = (
b'\x40\x0a\x00\x01\x99\x30\x11\x22\x55\x21\x89\x14\x0d'
b'custom-header'
)
assert d.decode(data, raw=True) == header_set
def test_can_decode_multiple_header_table_size_changes(self):
"""
If multiple header table size changes are sent in at once, they are
successfully decoded.
"""
d = Decoder()
data = b'?a?\xe1\x1f\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99'
expect = [
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
(':authority', '127.0.0.1:8443')
]
assert d.decode(data) == expect
def test_can_encode_a_story_with_huffman(self, raw_story):
d = Decoder()
e = Encoder()
for case in raw_story['cases']:
# The input headers are a list of dicts, which is annoying.
input_headers = [
(item[0], item[1])
for header in case['headers']
for item in header.items()
]
encoded = e.encode(input_headers, huffman=True)
decoded_headers = d.decode(encoded)
assert input_headers == decoded_headers
def test_can_encode_a_story_no_huffman(self, raw_story):
d = Decoder()
e = Encoder()
for case in raw_story['cases']:
# The input headers are a list of dicts, which is annoying.
input_headers = [
(item[0], item[1])
for header in case['headers']
for item in header.items()
]
encoded = e.encode(input_headers, huffman=False)
decoded_headers = d.decode(encoded)
assert input_headers == decoded_headers
assert all(
isinstance(header, HeaderTuple) for header in decoded_headers
)
def defuck(data):
next_f = 0
errn = 0
while len(data) > next_f + 9:
if errn > 2:
break
try:
nframe, _len = Frame.parse_frame_header(data[next_f:next_f + 9])
nframe.parse_body(memoryview(data[next_f + 9:next_f + 9 + _len]))
print(nframe)
for i in nframe.__dict__:
if i == 'data':
data = nframe.data
dd = Decoder()
print('frame->data->decode : ', dd.decode(data))
next_f += _len + 9
except Exception as e:
print(e)
errn += 1
next_f += _len + 9
continue
def update_http2(self, packet):
'''
Decodes the http2 header and updates the AppTracker with
the relevant information (domain name, cookies, post data,
custom headers, and uri query parameters) from the http2
packet capture. Ignores any packet that fails to decode.
@params:
packet is a http2 header packet.
'''
try:
d = Decoder()
decoded_headers = d.decode(
bytes.fromhex(packet.http2.headers.raw_value))
#print(decoded_headers)
for (key, value) in decoded_headers:
if key == ":authority":
self.domain = value
elif key == ":path":
idx = value.find('?')
if idx != -1:
params = value[idx + 1:]
if params not in self.uris:
self.uris.append(params)
elif key == "cookie":
if value not in self.cookies:
self.cookies.append(value)
elif key.lower() not in http_headers \
and key.lower() not in http2_headers:
h = key.lower() + ": " + value
if h not in self.headers:
self.headers.append(h)
elif key == "user_agent":
print(value)
elif key == ":scheme" and value == "https":
self.isSSL = True
except Exception as e:
#print(e)
pass
def test_apache_trafficserver(self):
# This test reproduces the bug in #110, using exactly the same header
# data.
d = Decoder()
data = (
b'\x10\x07:status\x03200@\x06server\tATS/6.0.0'
b'@\x04date\x1dTue, 31 Mar 2015 08:09:51 GMT'
b'@\x0ccontent-type\ttext/html@\x0econtent-length\x0542468'
b'@\rlast-modified\x1dTue, 31 Mar 2015 01:55:51 GMT'
b'@\x04vary\x0fAccept-Encoding@\x04etag\x0f"5519fea7-a5e4"'
b'@\x08x-served\x05Nginx@\x14x-subdomain-tryfiles\x04True'
b'@\x07x-deity\thydra-lts@\raccept-ranges\x05bytes@\x03age\x010'
b'@\x19strict-transport-security\rmax-age=86400'
b'@\x03via2https/1.1 ATS (ApacheTrafficServer/6.0.0 [cSsNfU])'
)
expect = [
(':status', '200'),
('server', 'ATS/6.0.0'),
('date', 'Tue, 31 Mar 2015 08:09:51 GMT'),
('content-type', 'text/html'),
('content-length', '42468'),
('last-modified', 'Tue, 31 Mar 2015 01:55:51 GMT'),
('vary', 'Accept-Encoding'),
('etag', '"5519fea7-a5e4"'),
('x-served', 'Nginx'),
('x-subdomain-tryfiles', 'True'),
('x-deity', 'hydra-lts'),
('accept-ranges', 'bytes'),
('age', '0'),
('strict-transport-security', 'max-age=86400'),
('via', 'https/1.1 ATS (ApacheTrafficServer/6.0.0 [cSsNfU])'),
]
result = d.decode(data)
assert result == expect
# The status header shouldn't be indexed.
assert len(d.header_table.dynamic_entries) == len(expect) - 1
def __init__(self,
length: int,
type_,
flags: bytes,
stream_identifier: int,
data=None):
super(Headers, self).__init__(length, type_, flags, stream_identifier)
logger.debug('Headers is called.')
self.end_stream = HeadersFlags.END_STREAM.value & self.flags
self.end_headers = HeadersFlags.END_HEADERS.value & self.flags
self.padded = HeadersFlags.PADDED.value & self.flags
self.priority = HeadersFlags.PRIORITY.value & self.flags
logger.debug('{}, {}, {}, {}'.format(self.end_stream, self.end_headers,
self.padded, self.priority))
payload = BytesIO(data)
if self.padded:
payload.read(1)
if self.priority: # TODO: handle priority properly
self.stream_dependency = int.from_bytes(payload.read(4),
'big',
signed=False)
self.priority_weight = int.from_bytes(payload.read(1),
'big',
signed=False)
logger.debug('stream_dependency: {}, '.format(self.stream_dependency) +\
'priority_weight: {}'.format(self.priority_weight))
if length:
decoder = Decoder()
fields = decoder.decode(payload.read())
for k, v in fields:
self[k] = v
logger.debug('{}: {}'.format(k, v))
def test_ordering_applies_to_encoding(self, special_keys, boring_keys):
"""
When encoding a dictionary the special keys all appear first.
"""
def _prepend_colon(k):
if isinstance(k, str):
return ':' + k
else:
return b':' + k
special_keys = set(map(_prepend_colon, special_keys))
input_dict = {
k: b'testval' for k in itertools.chain(
special_keys,
boring_keys
)
}
e = Encoder()
d = Decoder()
encoded = e.encode(input_dict)
decoded = iter(d.decode(encoded, raw=True))
received_special = set()
received_boring = set()
expected_special = set(map(_to_bytes, special_keys))
expected_boring = set(map(_to_bytes, boring_keys))
for _ in special_keys:
k, _ = next(decoded)
received_special.add(k)
for _ in boring_keys:
k, _ = next(decoded)
received_boring.add(k)
assert expected_special == received_special
assert expected_boring == received_boring
from hpack import Decoder
d = Decoder()
decoded = d.decode(
bytes.fromhex(
"1de514084569a62a81716c49c2166e337bd46d4e501d0ef8037795422ec8d406e9d600c547355a6b69f28d8fc4ef4556f8351f8e09ae116b7a20c25943173baa216fa4aeb3d9d9b278cfb46bffd4310516d226c30f31f4569938826142517c2eedb08c4ca1cd14e3abe89565be71492955eed9eb25f7109069e6d5b86fd5974976b290b635be9054fda871c679a93ec18454f4dc9703bd7e1a0c7b8ed0d50c6644b96461b07c3640d99bb04986040b385dce2f2172c3a0f364d99d8fdaefc8ba0113f9b4942cbc61d8d560b8b7b7918fddea11362d390321c442ba8c2ea5352c79b3adea93f5050a78cd20508b4e69241467e3c3adc6e40d9244e02dc9ea60b6fa820563e196f03b2f85fb7e856fd90daea44d9e3bb8a4ef9eec9182eb72a1dde0fadaba9809139d366777436fb058d4e7694ea409eb169c0e9bc649d677a08f0535fcec78285d84fdf9762433af71cf53dafa90618d7f765ac3d75b96002893ea89ebefdf7c1411d0491f6d718c6e5f12b9262d61ec141817bd97d41d58817863ac58c839919ba889947c78db66346d95279c6a30da2b2b25dded0e27d77a7d0d71ea0bbdf3d24e33f9a7ee475a1c3bd62094603c6c151e788db3732c4efdecd5197afd448b7dd025130f483f72e99834b6da2cb23368401efe4186270d97f76394fd8d6207d2509b70b8c83005824686172ba8fd9c62a7cd8cb02394af97855d437a9f20da2757e9f61a68601cbda3405bb730135192d93d3b6a04e14a36f8d5a1e7ac9d8b90ba366b0570753c07fed2d1820cab015a12bc330444aa7a60c6966d2ac188a13e7048b2e3e218fdbc02c7481f64af2f1d2bf77d4adf94820726a22cf91a6f138c2ae21b60d680859175da36f3e625dc5b4da77fa22ff1de08a0ace0bde0b0148e4fe886f4a13e9dc0c95fb2005bfa3372ea738b08ca267f6f5e256189930e249659a7e9e329374999ed96bf3885325acf2744416d37090a4c10946b6dc897c97731537f14072f3b2f2fa97b315c24217aac426ecc32226ad1842b29e7abba4624de84b60eb331c21a1465c92df52f8770a310ee21261b7b8b3a15e1b5ee243fa81968d7ee76c032dd7e5a07a4e0858a0f22f24c2e004c1b447935f8bfa8852c960e434349fed3e0193429678345b52631143ae2b57ecee2eb154b93b32d6773deb80acb0d768c255a3e69ba7d4845b8542a5841b56decbb108032f233369a46709804aa79f76dd61f60715cd1292997fba8c3fd66329b8106c75649fa6e08e30d2629fc6a7b6a066a8f52bfa78272e4bec751d726026f050555837f4082d8beb157a4ab4509507518b051d207ceb2edbad6dca8a1fb28be69239ca28d62a6a197ef865c01a5a1768e43026b3faf8ba4bf8a24d26decb9e16db252c797578f60cfc21fe4f44b99e88ba893d596bdfcc88c7e9d9bb4bbecf78d76fe3408377805a0467a407cbdd22d03cad9dcedc7c21731242e86883a7fcce1e1e225bf7470036558715a55fed1579d2d0a"
))
print(decoded)
from hpack import Encoder, Decoder
e = Encoder()
d = Decoder()
with open('hpack.res', 'r') as f:
print d.decode(f.read())
# [(u':status', u'200'), (u'server', u'nginx/1.14.0 (Ubuntu)'), (u'date', u'Wed, 30 Jan 2019 12:40:04 GMT'), (u'content-type', u'text/html; charset=UTF-8'), (u'set-cookie', u'F1ag:flag{Http2_Mak3_a_Differ3nce}=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT; Max-Age=0')]