def test_indexed_header_field(self):
"""
The header field representation uses an indexed header field, from
the static table.
"""
d = Decoder()
header_set = [(':method', 'GET')]
data = b'\x82'
assert d.decode(data) == header_set
assert list(d.header_table.dynamic_entries) == []
def test_literal_header_field_without_indexing(self):
"""
The header field representation uses an indexed name and a literal
value.
"""
d = Decoder()
header_set = [(':path', '/sample/path')]
data = b'\x04\x0c/sample/path'
assert d.decode(data) == header_set
assert list(d.header_table.dynamic_entries) == []
def save(self):
encoder = Encoder()
payload = encoder.encode(self)
self.length = len(payload)
decoder = Decoder()
check = decoder.decode(payload)
logger.info('payload of the header is {}'.format(check))
base = super().save()
return base + payload
def test_literal_never_indexed_emits_neverindexedheadertuple(self):
"""
A literal header field that must never be indexed emits a
NeverIndexedHeaderTuple.
"""
d = Decoder()
data = b'\x10\x0acustom-key\x0dcustom-header'
headers = d.decode(data)
assert len(headers) == 1
header = headers[0]
assert isinstance(header, NeverIndexedHeaderTuple)
def test_indexed_never_indexed_emits_neverindexedheadertuple(self):
"""
A header field with an indexed name that must never be indexed emits a
NeverIndexedHeaderTuple.
"""
d = Decoder()
data = b'\x14\x0c/sample/path'
headers = d.decode(data)
assert len(headers) == 1
header = headers[0]
assert isinstance(header, NeverIndexedHeaderTuple)
def test_literal_header_field_with_indexing_emits_headertuple(self):
"""
A header field with indexing emits a HeaderTuple.
"""
d = Decoder()
data = b'\x00\x0acustom-key\x0dcustom-header'
headers = d.decode(data)
assert len(headers) == 1
header = headers[0]
assert isinstance(header, HeaderTuple)
assert not isinstance(header, NeverIndexedHeaderTuple)
def test_literal_header_field_with_indexing(self):
"""
The header field representation uses a literal name and a literal
value.
"""
d = Decoder()
header_set = [('custom-key', 'custom-header')]
data = b'\x40\x0acustom-key\x0dcustom-header'
assert d.decode(data) == header_set
assert list(d.header_table.dynamic_entries) == [
(n.encode('utf-8'), v.encode('utf-8')) for n, v in header_set
]
def test_request_examples_with_huffman(self):
"""
This section shows the same examples as the previous section, but
using Huffman encoding for the literal values.
"""
d = Decoder()
first_header_set = [
(':method', 'GET',),
(':scheme', 'http',),
(':path', '/',),
(':authority', 'www.example.com'),
]
first_data = (
b'\x82\x86\x84\x01\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0\xab\x90\xf4\xff'
)
assert d.decode(first_data) == first_header_set
assert list(d.header_table.dynamic_entries) == []
second_header_set = [
(':method', 'GET',),
(':scheme', 'http',),
(':path', '/',),
(':authority', 'www.example.com',),
('cache-control', 'no-cache'),
]
second_data = (
b'\x82\x86\x84\x01\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0\xab\x90\xf4\xff'
b'\x0f\t\x86\xa8\xeb\x10d\x9c\xbf'
)
assert d.decode(second_data) == second_header_set
assert list(d.header_table.dynamic_entries) == []
third_header_set = [
(':method', 'GET',),
(':scheme', 'https',),
(':path', '/index.html',),
(':authority', 'www.example.com',),
('custom-key', 'custom-value'),
]
third_data = (
b'\x82\x87\x85\x01\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0\xab\x90\xf4\xff@'
b'\x88%\xa8I\xe9[\xa9}\x7f\x89%\xa8I\xe9[\xb8\xe8\xb4\xbf'
)
assert d.decode(third_data) == third_header_set
assert len(d.header_table.dynamic_entries) == 1
def test_request_examples_without_huffman(self):
"""
This section shows several consecutive header sets, corresponding to
HTTP requests, on the same connection.
"""
d = Decoder()
first_header_set = [
(':method', 'GET',),
(':scheme', 'http',),
(':path', '/',),
(':authority', 'www.example.com'),
]
# The first_header_table doesn't contain 'authority'
first_data = b'\x82\x86\x84\x01\x0fwww.example.com'
assert d.decode(first_data) == first_header_set
assert list(d.header_table.dynamic_entries) == []
# This request takes advantage of the differential encoding of header
# sets.
second_header_set = [
(':method', 'GET',),
(':scheme', 'http',),
(':path', '/',),
(':authority', 'www.example.com',),
('cache-control', 'no-cache'),
]
second_data = (
b'\x82\x86\x84\x01\x0fwww.example.com\x0f\t\x08no-cache'
)
assert d.decode(second_data) == second_header_set
assert list(d.header_table.dynamic_entries) == []
third_header_set = [
(':method', 'GET',),
(':scheme', 'https',),
(':path', '/index.html',),
(':authority', 'www.example.com',),
('custom-key', 'custom-value'),
]
third_data = (
b'\x82\x87\x85\x01\x0fwww.example.com@\ncustom-key\x0ccustom-value'
)
assert d.decode(third_data) == third_header_set
# Don't check the header table here, it's just too complex to be
# reliable. Check its length though.
assert len(d.header_table.dynamic_entries) == 1
def test_can_decode_multiple_header_table_size_changes(self):
"""
If multiple header table size changes are sent in at once, they are
successfully decoded.
"""
d = Decoder()
data = b'?a?\xe1\x1f\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99'
expect = [
(':method', 'GET'),
(':scheme', 'https'),
(':path', '/'),
(':authority', '127.0.0.1:8443')
]
assert d.decode(data) == expect
def test_raw_decoding(self):
"""
The header field representation is decoded as a raw byte string instead
of UTF-8
"""
d = Decoder()
header_set = [
(b'\x00\x01\x99\x30\x11\x22\x55\x21\x89\x14', b'custom-header')
]
data = (
b'\x40\x0a\x00\x01\x99\x30\x11\x22\x55\x21\x89\x14\x0d'
b'custom-header'
)
assert d.decode(data, raw=True) == header_set
def test_truncated_header_name(self):
"""
If a header name is truncated an error is raised.
"""
d = Decoder()
# This is a simple header block that has a bad ending. The interesting
# part begins on the second line. This indicates a string that has
# literal name and value. The name is a 5 character huffman-encoded
# string that is only three bytes long.
data = (
b'\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99'
b'\x00\x85\xf2\xb2J'
)
with pytest.raises(HPACKDecodingError):
d.decode(data)
def test_can_encode_a_story_with_huffman(self, raw_story):
d = Decoder()
e = Encoder()
for case in raw_story['cases']:
# The input headers are a list of dicts, which is annoying.
input_headers = [
(item[0], item[1])
for header in case['headers']
for item in header.items()
]
encoded = e.encode(input_headers, huffman=True)
decoded_headers = d.decode(encoded)
assert input_headers == decoded_headers
class HeadersFrame(Frame):
type = 0x1
allowed_flags = [
Flag('END_STREAM', 0x1),
Flag('END_HEADERS', 0x4),
Flag('PADDED', 0x8),
Flag('PRIORITY', 0x20)
]
header_decoder = Decoder()
def __init__(self, stream_id, data=b'', **kwargs):
super().__init__(stream_id, **kwargs)
self.data = data
def parse_payload(self, payload):
pos = 0
if 'PADDED' in self.flags:
self.pad_length = payload[pos]
pos += 1
else:
self.pad_length = 0
if 'PRIORITY' in self.flags:
self.stream_dependency = struct.unpack('!L',
payload[pos:pos + 4])[0]
self.exclusive = (self.stream_dependency >> 31) & 1
self.stream_dependency &= 0x7FFFFFFF
pos += 4
self.weight = payload[pos] + 1
pos += 1
self.header_block = payload[pos:len(payload) - self.pad_length]
self.headers = self.header_decoder.decode(self.header_block)
def serialize_payload(self):
padding_data = b''
padding = b'\0' * 0
if 'PRIORITY' in self.flags:
priority_data = b''
else:
priority_data = b''
return b''.join([padding_data, priority_data, self.data, padding])
class PushPromiseFrame(Frame):
type = 0x5
allowed_flags = [Flag('END_HEADERS', 0x4), Flag('PADDED', 0x8)]
header_decoder = Decoder()
def parse_payload(self, payload):
pos = 0
if 'PADDED' in self.flags:
self.pad_length = payload[pos]
pos += 1
else:
self.pad_length = 0
self.promised_stream_id = struct.unpack('!I', payload[pos:pos + 4])[0]
pos += 4
self.header_block = payload[pos:len(payload) - self.pad_length]
self.headers = self.header_decoder.decode(self.header_block)
def test_truncated_header_value(self):
"""
If a header value is truncated an error is raised.
"""
d = Decoder()
# This is a simple header block that has a bad ending. The interesting
# part begins on the second line. This indicates a string that has
# literal name and value. The name is a 5 character huffman-encoded
# string, but the entire EOS character has been written over the end.
# This causes hpack to see the header value as being supposed to be
# 622462 bytes long, which it clearly is not, and so this must fail.
data = (
b'\x82\x87\x84A\x8a\x08\x9d\\\x0b\x81p\xdcy\xa6\x99'
b'\x00\x85\xf2\xb2J\x87\xff\xff\xff\xfd%B\x7f'
)
with pytest.raises(HPACKDecodingError):
d.decode(data)
def test_can_encode_a_story_no_huffman(self, raw_story):
d = Decoder()
e = Encoder()
for case in raw_story['cases']:
# The input headers are a list of dicts, which is annoying.
input_headers = [
(item[0], item[1])
for header in case['headers']
for item in header.items()
]
encoded = e.encode(input_headers, huffman=False)
decoded_headers = d.decode(encoded)
assert input_headers == decoded_headers
assert all(
isinstance(header, HeaderTuple) for header in decoded_headers
)
def test_resizing_header_table(self):
# We need to decode a substantial number of headers, to populate the
# header table. This string isn't magic: it's the output from the
# equivalent test for the Encoder.
d = Decoder()
data = (
b'\x82\x87D\x87a\x07\xa4\xacV4\xcfA\x8c\xf1\xe3\xc2\xe5\xf2:k\xa0'
b'\xab\x90\xf4\xff@\x88%\xa8I\xe9[\xa9}\x7f\x89%\xa8I\xe9[\xb8\xe8'
b'\xb4\xbfz\xbc\xd0\x7ff\xa2\x81\xb0\xda\xe0S\xfa\xd02\x1a\xa4\x9d'
b'\x13\xfd\xa9\x92\xa4\x96\x854\x0c\x8aj\xdc\xa7\xe2\x81\x02\xef}'
b'\xa9g{\x81qp\x7fjb):\x9d\x81\x00 \x00@\x150\x9a\xc2\xca\x7f,\x05'
b'\xc5\xc1S\xb0I|\xa5\x89\xd3M\x1fC\xae\xba\x0cA\xa4\xc7\xa9\x8f3'
b'\xa6\x9a?\xdf\x9ah\xfa\x1du\xd0b\r&=Ly\xa6\x8f\xbe\xd0\x01w\xfe'
b'\xbeX\xf9\xfb\xed\x00\x17{@\x8a\xfc[=\xbdF\x81\xad\xbc\xa8O\x84y'
b'\xe7\xde\x7f'
)
d.decode(data)
# Resize the header table to a size so small that nothing can be in it.
d.header_table_size = 40
assert len(d.header_table.dynamic_entries) == 0
def defuck(data):
next_f = 0
errn = 0
while len(data) > next_f + 9:
if errn > 2:
break
try:
nframe, _len = Frame.parse_frame_header(data[next_f:next_f + 9])
nframe.parse_body(memoryview(data[next_f + 9:next_f + 9 + _len]))
print(nframe)
for i in nframe.__dict__:
if i == 'data':
data = nframe.data
dd = Decoder()
print('frame->data->decode : ', dd.decode(data))
next_f += _len + 9
except Exception as e:
print(e)
errn += 1
next_f += _len + 9
continue
def update_http2(self, packet):
'''
Decodes the http2 header and updates the AppTracker with
the relevant information (domain name, cookies, post data,
custom headers, and uri query parameters) from the http2
packet capture. Ignores any packet that fails to decode.
@params:
packet is a http2 header packet.
'''
try:
d = Decoder()
decoded_headers = d.decode(
bytes.fromhex(packet.http2.headers.raw_value))
#print(decoded_headers)
for (key, value) in decoded_headers:
if key == ":authority":
self.domain = value
elif key == ":path":
idx = value.find('?')
if idx != -1:
params = value[idx + 1:]
if params not in self.uris:
self.uris.append(params)
elif key == "cookie":
if value not in self.cookies:
self.cookies.append(value)
elif key.lower() not in http_headers \
and key.lower() not in http2_headers:
h = key.lower() + ": " + value
if h not in self.headers:
self.headers.append(h)
elif key == "user_agent":
print(value)
elif key == ":scheme" and value == "https":
self.isSSL = True
except Exception as e:
#print(e)
pass
def test_apache_trafficserver(self):
# This test reproduces the bug in #110, using exactly the same header
# data.
d = Decoder()
data = (
b'\x10\x07:status\x03200@\x06server\tATS/6.0.0'
b'@\x04date\x1dTue, 31 Mar 2015 08:09:51 GMT'
b'@\x0ccontent-type\ttext/html@\x0econtent-length\x0542468'
b'@\rlast-modified\x1dTue, 31 Mar 2015 01:55:51 GMT'
b'@\x04vary\x0fAccept-Encoding@\x04etag\x0f"5519fea7-a5e4"'
b'@\x08x-served\x05Nginx@\x14x-subdomain-tryfiles\x04True'
b'@\x07x-deity\thydra-lts@\raccept-ranges\x05bytes@\x03age\x010'
b'@\x19strict-transport-security\rmax-age=86400'
b'@\x03via2https/1.1 ATS (ApacheTrafficServer/6.0.0 [cSsNfU])'
)
expect = [
(':status', '200'),
('server', 'ATS/6.0.0'),
('date', 'Tue, 31 Mar 2015 08:09:51 GMT'),
('content-type', 'text/html'),
('content-length', '42468'),
('last-modified', 'Tue, 31 Mar 2015 01:55:51 GMT'),
('vary', 'Accept-Encoding'),
('etag', '"5519fea7-a5e4"'),
('x-served', 'Nginx'),
('x-subdomain-tryfiles', 'True'),
('x-deity', 'hydra-lts'),
('accept-ranges', 'bytes'),
('age', '0'),
('strict-transport-security', 'max-age=86400'),
('via', 'https/1.1 ATS (ApacheTrafficServer/6.0.0 [cSsNfU])'),
]
result = d.decode(data)
assert result == expect
# The status header shouldn't be indexed.
assert len(d.header_table.dynamic_entries) == len(expect) - 1
def genFrame(data):
next_f = 0
errn = 0
while len(data) > next_f+9:
print("[*] "+"-"*30)
if errn > 2:
break
try:
nframe, _len = Frame.parse_frame_header(
data[next_f:next_f+9])
nframe.parse_body(memoryview(data[next_f+9:next_f+9 + _len]))
print("[+] Frame -> %s" % nframe)
for i in nframe.__dict__:
if i == 'data':
print("[+] Data:")
print("[+] ", Decoder().decode(nframe.data))
print("[+] ")
next_f += _len + 9
except Exception as e:
print(e)
errn += 1
next_f += _len + 9
continue
def __init__(self,
length: int,
type_,
flags: bytes,
stream_identifier: int,
data=None):
super(Headers, self).__init__(length, type_, flags, stream_identifier)
logger.debug('Headers is called.')
self.end_stream = HeadersFlags.END_STREAM.value & self.flags
self.end_headers = HeadersFlags.END_HEADERS.value & self.flags
self.padded = HeadersFlags.PADDED.value & self.flags
self.priority = HeadersFlags.PRIORITY.value & self.flags
logger.debug('{}, {}, {}, {}'.format(self.end_stream, self.end_headers,
self.padded, self.priority))
payload = BytesIO(data)
if self.padded:
payload.read(1)
if self.priority: # TODO: handle priority properly
self.stream_dependency = int.from_bytes(payload.read(4),
'big',
signed=False)
self.priority_weight = int.from_bytes(payload.read(1),
'big',
signed=False)
logger.debug('stream_dependency: {}, '.format(self.stream_dependency) +\
'priority_weight: {}'.format(self.priority_weight))
if length:
decoder = Decoder()
fields = decoder.decode(payload.read())
for k, v in fields:
self[k] = v
logger.debug('{}: {}'.format(k, v))
def test_ordering_applies_to_encoding(self, special_keys, boring_keys):
"""
When encoding a dictionary the special keys all appear first.
"""
def _prepend_colon(k):
if isinstance(k, str):
return ':' + k
else:
return b':' + k
special_keys = set(map(_prepend_colon, special_keys))
input_dict = {
k: b'testval' for k in itertools.chain(
special_keys,
boring_keys
)
}
e = Encoder()
d = Decoder()
encoded = e.encode(input_dict)
decoded = iter(d.decode(encoded, raw=True))
received_special = set()
received_boring = set()
expected_special = set(map(_to_bytes, special_keys))
expected_boring = set(map(_to_bytes, boring_keys))
for _ in special_keys:
k, _ = next(decoded)
received_special.add(k)
for _ in boring_keys:
k, _ = next(decoded)
received_boring.add(k)
assert expected_special == received_special
assert expected_boring == received_boring
def __init__(self, logger, ip_manager, config, ssl_sock, close_cb,
retry_task_cb, idle_cb, log_debug_data):
super(Http2Worker,
self).__init__(logger, ip_manager, config, ssl_sock, close_cb,
retry_task_cb, idle_cb, log_debug_data)
self.network_buffer_size = 65535
# Google http/2 time out is 4 mins.
self.ssl_sock.settimeout(240)
self._sock = BufferedSocket(ssl_sock, self.network_buffer_size)
self.next_stream_id = 1
self.streams = {}
self.last_ping_time = time.time()
self.continue_timeout = 0
# count ping not ACK
# increase when send ping
# decrease when recv ping ack
# if this in not 0, don't accept request.
self.ping_on_way = 0
self.accept_task = False
# request_lock
self.request_lock = threading.Lock()
# all send frame must put to this queue
# then send by send_loop
# every frame put to this queue must allowed by stream window and connection window
# any data frame blocked by connection window should put to self.blocked_send_frames
self.send_queue = Queue.Queue()
self.encoder = Encoder()
self.decoder = Decoder()
# keep blocked data frame in this buffer
# which is allowed by stream window but blocked by connection window.
# They will be sent when connection window open
self.blocked_send_frames = []
# Values for the settings used on an HTTP/2 connection.
# will send to remote using Setting Frame
self.local_settings = {
SettingsFrame.INITIAL_WINDOW_SIZE: 16 * 1024 * 1024,
SettingsFrame.SETTINGS_MAX_FRAME_SIZE: 256 * 1024
}
self.local_connection_initial_windows = 32 * 1024 * 1024
self.local_window_manager = FlowControlManager(
self.local_connection_initial_windows)
# changed by server, with SettingFrame
self.remote_settings = {
SettingsFrame.INITIAL_WINDOW_SIZE: DEFAULT_WINDOW_SIZE,
SettingsFrame.SETTINGS_MAX_FRAME_SIZE: DEFAULT_MAX_FRAME,
SettingsFrame.MAX_CONCURRENT_STREAMS: 100
}
#self.remote_window_size = DEFAULT_WINDOW_SIZE
self.remote_window_size = 32 * 1024 * 1024
# send Setting frame before accept task.
self._send_preamble()
threading.Thread(target=self.send_loop).start()
threading.Thread(target=self.recv_loop).start()
def __init__(self):
super().__init__()
self.decoder = Decoder()
self._factory = {klass.FrameType(): klass for klass in FrameBase.__subclasses__()}
from hpack import Decoder
d = Decoder()
decoded = d.decode(
bytes.fromhex(
"1de514084569a62a81716c49c2166e337bd46d4e501d0ef8037795422ec8d406e9d600c547355a6b69f28d8fc4ef4556f8351f8e09ae116b7a20c25943173baa216fa4aeb3d9d9b278cfb46bffd4310516d226c30f31f4569938826142517c2eedb08c4ca1cd14e3abe89565be71492955eed9eb25f7109069e6d5b86fd5974976b290b635be9054fda871c679a93ec18454f4dc9703bd7e1a0c7b8ed0d50c6644b96461b07c3640d99bb04986040b385dce2f2172c3a0f364d99d8fdaefc8ba0113f9b4942cbc61d8d560b8b7b7918fddea11362d390321c442ba8c2ea5352c79b3adea93f5050a78cd20508b4e69241467e3c3adc6e40d9244e02dc9ea60b6fa820563e196f03b2f85fb7e856fd90daea44d9e3bb8a4ef9eec9182eb72a1dde0fadaba9809139d366777436fb058d4e7694ea409eb169c0e9bc649d677a08f0535fcec78285d84fdf9762433af71cf53dafa90618d7f765ac3d75b96002893ea89ebefdf7c1411d0491f6d718c6e5f12b9262d61ec141817bd97d41d58817863ac58c839919ba889947c78db66346d95279c6a30da2b2b25dded0e27d77a7d0d71ea0bbdf3d24e33f9a7ee475a1c3bd62094603c6c151e788db3732c4efdecd5197afd448b7dd025130f483f72e99834b6da2cb23368401efe4186270d97f76394fd8d6207d2509b70b8c83005824686172ba8fd9c62a7cd8cb02394af97855d437a9f20da2757e9f61a68601cbda3405bb730135192d93d3b6a04e14a36f8d5a1e7ac9d8b90ba366b0570753c07fed2d1820cab015a12bc330444aa7a60c6966d2ac188a13e7048b2e3e218fdbc02c7481f64af2f1d2bf77d4adf94820726a22cf91a6f138c2ae21b60d680859175da36f3e625dc5b4da77fa22ff1de08a0ace0bde0b0148e4fe886f4a13e9dc0c95fb2005bfa3372ea738b08ca267f6f5e256189930e249659a7e9e329374999ed96bf3885325acf2744416d37090a4c10946b6dc897c97731537f14072f3b2f2fa97b315c24217aac426ecc32226ad1842b29e7abba4624de84b60eb331c21a1465c92df52f8770a310ee21261b7b8b3a15e1b5ee243fa81968d7ee76c032dd7e5a07a4e0858a0f22f24c2e004c1b447935f8bfa8852c960e434349fed3e0193429678345b52631143ae2b57ecee2eb154b93b32d6773deb80acb0d768c255a3e69ba7d4845b8542a5841b56decbb108032f233369a46709804aa79f76dd61f60715cd1292997fba8c3fd66329b8106c75649fa6e08e30d2629fc6a7b6a066a8f52bfa78272e4bec751d726026f050555837f4082d8beb157a4ab4509507518b051d207ceb2edbad6dca8a1fb28be69239ca28d62a6a197ef865c01a5a1768e43026b3faf8ba4bf8a24d26decb9e16db252c797578f60cfc21fe4f44b99e88ba893d596bdfcc88c7e9d9bb4bbecf78d76fe3408377805a0467a407cbdd22d03cad9dcedc7c21731242e86883a7fcce1e1e225bf7470036558715a55fed1579d2d0a"
))
print(decoded)
import logging
from logging.config import dictConfig
import pyshark
from h2.config import H2Configuration
from h2.connection import H2Connection
from h2.frame_buffer import FrameBuffer
from h2.frame_buffer import HeadersFrame
from hyperframe.frame import *
from hpack import Decoder
log_conf = json.load(open("logging.json"))
dictConfig(log_conf)
logger = logging.getLogger()
# HTTP2 Header decoder
decoder = Decoder()
# Change INPUT_CONFIG to your pcap file's name and server/client's IP address
INPUT_CONFIG = {
"pcap": "./pcap-sample/z.pcap",
"client": "10.20.50.25",
"server": "10.20.50.34",
}
pcap_file = INPUT_CONFIG["pcap"]
client_ip = INPUT_CONFIG["client"]
server_ip = INPUT_CONFIG["server"]
logger.debug("Input config: {0}".format(INPUT_CONFIG))
# Change STREAM_NUMBER to the stream number you want to follow.
# STREAM_NUMBER = 0
# Switching protocol check
def test_header_table_size_getter(self):
d = Decoder()
assert d.header_table_size