def unpack_name(buf, off):
name = []
name_length = 0
saved_off = 0
start_off = off
while True:
if off >= len(buf):
raise dpkt.NeedData()
n = ord(buf[off])
if n == 0:
off += 1
break
elif (n & 0xc0) == 0xc0:
ptr = struct.unpack('>H', buf[off:off + 2])[0] & 0x3fff
if ptr >= start_off:
raise dpkt.UnpackError('Invalid label compression pointer')
off += 2
if not saved_off:
saved_off = off
start_off = off = ptr
elif (n & 0xc0) == 0x00:
off += 1
name.append(buf[off:off + n])
name_length += n + 1
if name_length > 255:
raise dpkt.UnpackError('name longer than 255 bytes')
off += n
else:
raise dpkt.UnpackError('Invalid label length %02x' % n)
if not saved_off:
saved_off = off
return '.'.join(name), saved_off
def unpack(self, buf):
# TPKT header
self.tpkt = tpkt.TPKT(buf)
if self.tpkt.v != 3:
raise dpkt.UnpackError('invalid TPKT version')
if self.tpkt.rsvd != 0:
raise dpkt.UnpackError('invalid TPKT reserved value')
n = self.tpkt.len - self.tpkt.__hdr_len__
if n > len(self.tpkt.data):
raise dpkt.UnpackError('invalid TPKT length')
buf = self.tpkt.data
# Q.931 payload
dpkt.Packet.unpack(self, buf)
buf = buf[self.__hdr_len__:]
self.ref_val = buf[:self.ref_len]
buf = buf[self.ref_len:]
self.type = struct.unpack('B', buf[:1])[0]
buf = buf[1:]
# Information Elements
l = []
while buf:
ie = self.IE(buf)
l.append(ie)
buf = buf[len(ie):]
self.data = l
def decode(buf):
"""Sleazy ASN.1 decoder.
Return list of (id, value) tuples from ASN.1 BER/DER encoded buffer.
"""
msg = []
while buf:
t = ord(buf[0])
constructed = t & CONSTRUCTED
tag = t & TAGMASK
l = ord(buf[1])
c = 0
if constructed and l == 128:
# XXX - constructed, indefinite length
msg.append((t, decode(buf[2:])))
elif l >= 128:
c = l & 127
if c == 1:
l = ord(buf[2])
elif c == 2:
l = struct.unpack('>H', buf[2:4])[0]
elif c == 3:
l = struct.unpack('>I', buf[1:5])[0] & 0xfff
c = 2
elif c == 4:
l = struct.unpack('>I', buf[2:6])[0]
else:
# XXX - can be up to 127 bytes, but...
raise dpkt.UnpackError('excessive long-form ASN.1 length %d' %
l)
# Skip type, length
buf = buf[2 + c:]
# Parse content
if constructed:
msg.append((t, decode(buf)))
elif tag == INTEGER:
if l == 0:
n = 0
elif l == 1:
n = ord(buf[0])
elif l == 2:
n = struct.unpack('>H', buf[:2])[0]
elif l == 3:
n = struct.unpack('>I', buf[:4])[0] >> 8
elif l == 4:
n = struct.unpack('>I', buf[:4])[0]
else:
raise dpkt.UnpackError('excessive integer length > %d bytes' %
l)
msg.append((t, n))
elif tag == UTC_TIME:
msg.append((t, utctime(buf[:l])))
else:
msg.append((t, buf[:l]))
# Skip content
buf = buf[l:]
return msg
def unpack(self, buf):
f = BytesIO(buf)
line = f.readline().decode("ascii", "ignore")
l = line.strip().split()
if len(l) != 1:
raise dpkt.UnpackError('invalid message: %r' % line)
if l[0] not in self.__methods:
raise dpkt.UnpackError('invalid protocol method: %r' % l[0])
self.method = l[0]
Message.unpack(self, f.read())
def parse_body(f, headers):
"""Return HTTP body parsed from a file object, given HTTP header dict."""
if headers.get('transfer-encoding', '').lower() == 'chunked':
l = []
found_end = False
while 1:
try:
sz = f.readline().split(None, 1)[0]
except IndexError:
raise dpkt.UnpackError('missing chunk size')
n = int(sz, 16)
if n == 0:
found_end = True
buf = f.read(n)
if f.readline().strip():
break
if n and len(buf) == n:
l.append(buf)
else:
break
if not found_end:
raise dpkt.NeedData('premature end of chunked body')
body = ''.join(l)
elif 'content-length' in headers:
n = int(headers['content-length'])
body = f.read(n)
if len(body) != n:
raise dpkt.NeedData('short body (missing %d bytes)' %
(n - len(body)))
elif 'content-type' in headers:
body = f.read()
else:
# XXX - need to handle HTTP/0.9
body = ''
return body
def unpack(self, buf):
dpkt.Packet.unpack(self, buf)
if self.ast != 0x2a:
raise dpkt.UnpackError('invalid FLAP header')
if len(self.data) < self.len:
raise dpkt.NeedData, '%d left, %d needed' % (len(
self.data), self.len)
def unpack_rdata(self, buf, off):
if self.type == DNS_A:
self.ip = self.rdata
elif self.type == DNS_NS:
self.nsname, off = unpack_name(buf, off)
elif self.type == DNS_CNAME:
self.cname, off = unpack_name(buf, off)
elif self.type == DNS_PTR:
self.ptrname, off = unpack_name(buf, off)
elif self.type == DNS_SOA:
self.mname, off = unpack_name(buf, off)
self.rname, off = unpack_name(buf, off)
self.serial, self.refresh, self.retry, self.expire, \
self.minimum = struct.unpack('>IIIII', buf[off:off + 20])
elif self.type == DNS_MX:
self.preference = struct.unpack('>H', self.rdata[:2])
self.mxname, off = unpack_name(buf, off + 2)
elif self.type == DNS_TXT or self.type == DNS_HINFO:
self.text = []
buf = self.rdata
while buf:
n = ord(buf[0])
self.text.append(buf[1:1 + n])
buf = buf[1 + n:]
elif self.type == DNS_AAAA:
self.ip6 = self.rdata
elif self.type == DNS_SRV:
self.priority, self.weight, self.port = struct.unpack(
'>HHH', self.rdata[:6])
self.srvname, off = unpack_name(buf, off + 6)
elif self.type == DNS_OPT:
pass # RFC-6891: OPT is a pseudo-RR not carrying any DNS data
else:
raise dpkt.UnpackError('RR type %s is not supported' %
self.type)
def parse_headers(f):
"""Return dict of headers parsed from a file object."""
d = OrderedDict()
while 1:
# The following logic covers two kinds of loop exit criteria.
# 1) If the header is valid, when we reached the end of the header,
# f.readline() would return with '\r\n', then after strip(),
# we can break the loop.
# 2) If this is a weird header, which do not ends with '\r\n',
# f.readline() would return with '', then after strip(),
# we still get an empty string, also break the loop.
line = f.readline().strip().decode("ascii", "ignore")
if not line:
break
l = line.split(':', 1)
if len(l[0].split()) != 1:
raise dpkt.UnpackError('invalid header: %r' % line)
k = l[0].lower()
v = len(l) != 1 and l[1].lstrip() or ''
if k in d:
if not type(d[k]) is list:
d[k] = [d[k]]
d[k].append(v)
else:
d[k] = v
return d
def unpack(self, buf):
f = cStringIO.StringIO(buf)
line = f.readline()
l = line.strip().split()
if len(l) < 2:
raise dpkt.UnpackError('invalid request: %r' % line)
if l[0] not in self.__methods:
raise dpkt.UnpackError('invalid http method: %r' % l[0])
if len(l) == 2:
# HTTP/0.9 does not specify a version in the request line
self.version = '0.9'
else:
if not l[2].startswith(self.__proto):
raise dpkt.UnpackError('invalid http version: %r' % l[2])
self.version = l[2][len(self.__proto) + 1:]
self.method = l[0]
self.uri = l[1]
Message.unpack(self, f.read())
def unpack(self, buf):
f = cStringIO.StringIO(buf)
line = f.readline()
l = line.strip().split(None, 2)
if len(l) < 2 or not l[0].startswith(self.__proto) or not l[1].isdigit():
raise dpkt.UnpackError('invalid response: %r' % line)
self.version = l[0][len(self.__proto) + 1:]
self.status = l[1]
self.reason = l[2]
Message.unpack(self, f.read())
def unpack(self, buf):
f = cStringIO.StringIO(buf)
line = f.readline()
l = line.strip().split()
if len(l) != 3 or l[0] not in self.__methods or \
not l[2].startswith(self.__proto):
raise dpkt.UnpackError('invalid request: %r' % line)
self.method = l[0]
self.uri = l[1]
self.version = l[2][len(self.__proto) + 1:]
Message.unpack(self, f.read())
def unpack(self, buf):
#f = io.StringIO(str(buf)[2:-1])
f = io.BytesIO(buf)
line = f.readline()
line = line.decode('utf-8')
l = line.strip().split(None, 2)
if len(l) < 3 or not l[0].startswith(
self.__proto) or not l[1].isdigit():
raise dpkt.UnpackError('invalid response: %r' % line)
self.version = l[0][len(self.__proto) + 1:]
self.status = l[1]
self.reason = l[2]
parse_message(self, f)
def unpack(self, buf):
#f = io.StringIO(str(buf)[2:-1])
f = io.BytesIO(buf)
line = f.readline()
line = line.decode('utf-8')
l = line.strip().split()
if len(l) != 3 or l[0] not in self.__methods or \
not l[2].startswith(self.__proto):
raise dpkt.UnpackError('invalid request: %r' % line)
self.method = l[0]
self.uri = l[1]
self.version = l[2][len(self.__proto) + 1:]
parse_message(self, f)
def unpack(self, buf):
f = BytesIO(buf)
line = f.readline().decode("ascii", "ignore")
if line.startswith('HTTP/'):
# Work around needed to process pages where HTTP method is not specified. Mainly GET requests.
line = 'GET * ' + line
line = line.strip().split()
if len(line) < 2:
raise dpkt.UnpackError('invalid request: %r' % line)
if line[0] not in self.__methods:
raise dpkt.UnpackError('invalid http method: %r' % line[0])
if len(line) == 2:
# HTTP/0.9 does not specify a version in the request line
self.version = '0.9'
else:
if not line[2].startswith(self.__proto):
raise dpkt.UnpackError('invalid http version: %r' % line[2])
self.version = line[2][len(self.__proto) + 1:]
self.method = line[0]
self.uri = line[1]
Message.unpack(self, f.read())
def parse_body(f, version, headers):
"""Return HTTP body parsed from a file object, given HTTP header dict."""
if headers.get('transfer-encoding', '').lower() == 'chunked':
l = []
found_end = False
while 1:
try:
sz = f.readline().split(None, 1)[0]
except IndexError:
raise dpkt.UnpackError('missing chunk size')
n = parse_length(sz, 16)
if n == 0: # may happen if sz is invalid
found_end = True
buf = f.read(n)
if f.readline().strip():
break
if n and len(buf) == n:
l.append(buf)
else:
break
if settings.strict_http_parse_body and not found_end:
raise dpkt.NeedData('premature end of chunked body')
body = ''.join(l)
elif 'content-length' in headers:
# Ethan K B: Have observed malformed 0,0 content lengths
n = parse_length(headers['content-length'])
body = f.read(n)
if len(body) != n:
logging.warn('HTTP content-length mismatch: expected %d, got %d', n,
len(body))
if settings.strict_http_parse_body:
raise dpkt.NeedData('short body (missing %d bytes)' % (n - len(body)))
else:
# XXX - need to handle HTTP/0.9
# BTW, this function is not called if status code is 204 or 304
if version == '1.0':
# we can assume that there are no further
# responses on this stream, since 1.0 doesn't
# support keepalive
body = f.read()
elif (version == '1.1' and
headers.get('connection', None) == 'close'):
# sender has said they won't send anything else.
body = f.read()
# there's also the case where other end sends connection: close,
# but we don't have the architecture to handle that.
else:
# we don't really know what to do
#print 'returning body as empty string:', version, headers
body = ''
return body
def _read_chunked(rfile):
"""
Read a HTTP body with chunked transfer encoding.
(adapted from mitmproxy's netlib.http.http1)
"""
while True:
line = rfile.readline(128)
if line == b"":
raise dpkt.NeedData("premature end of chunked body")
if line != b"\r\n" and line != b"\n":
try:
length = int(line, 16)
except ValueError:
raise dpkt.UnpackError("Invalid chunked encoding length: %s" %
line)
chunk = rfile.read(length)
suffix = rfile.readline(5)
if suffix != b"\r\n":
raise dpkt.UnpackError("Malformed chunked body")
if length == 0:
return
yield chunk
def unpack_addr(addrtype, buf, offset):
if addrtype == IP_V4:
addr = socket.inet_ntop(socket.AF_INET, buf[offset:(offset + 4)])
nxt = offset + 4
elif addrtype == IP_V6:
addr = socket.inet_ntop(socket.AF_INET6, buf[offset:(offset + 16)])
nxt = offset + 16
elif addrtype == DOMAIN_NAME:
length = struct.unpack('B', buf[offset])[0]
addr = buf[(offset + 1):(offset + 1 + length)]
nxt = offset + 1 + length
else:
raise dpkt.UnpackError("Unknown address type %s" %
addrtype.encode('hex'))
return addr, nxt
def parse_headers(f):
"""Return dict of HTTP headers parsed from a file object."""
d = {}
while 1:
line = f.readline()
if not line:
raise dpkt.NeedData('premature end of headers')
line = line.strip()
if not line:
break
l = line.split(None, 1)
if not l[0].endswith(':'):
raise dpkt.UnpackError('invalid header: %r' % line)
k = l[0][:-1].lower()
d[k] = len(l) != 1 and l[1] or ''
return d
def parse_body(self, file, headers, fix_incomplete=False):
"""Return HTTP body parsed from a file object, given HTTP header dict
Args:
file: BytesIO object
headers: HTTP request headers as dict
Returns:
str: body string"""
if headers.get('transfer-encoding', '').lower() == 'chunked':
buffer_list = []
found_end = False
while True:
try:
size = file.readline().split(None, 1)[0]
except IndexError:
raise dpkt.UnpackError('missing chunk size')
n = int(size, 16)
if n == 0:
found_end = True
buffer = file.read(n)
if file.readline().strip():
break
if n and len(buffer) == n:
buffer_list.append(buffer)
else:
break
if not found_end:
raise dpkt.NeedData('premature end of chunked body')
body = b''.join(buffer_list)
elif 'content-length' in headers:
n = int(headers['content-length'])
body = file.read(n)
if fix_incomplete:
headers['content-length'] = len(body)
elif len(body) != n:
raise dpkt.NeedData('short body (missing %d bytes)' %
(n - len(body)))
elif 'content-type' in headers:
body = file.read()
else:
# XXX - need to handle HTTP/0.9
body = b''
return body
def unpack(self, buf):
"""Unpacks a bytes object into component attributes.
This function is called when an instance of this class is created
by passing a bytes object to the constructor
"""
super().unpack(buf) # unpacks _hdr
if self.hdr_type != APv6Udp.HDR_TYPE:
raise dpkt.UnpackError(
"Unpacking compressed UDP, but encountered wrong type value")
if self.hdr_co == 0:
if len(self.data) < 2:
raise dpkt.NeedData("for UDP checksum")
self.chksum = struct.unpack("!H", self.data[0:2])[0]
self.data = self.data[2:]
p = self.hdr_ports
if p == APv6Udp.HDR_PORTS_SRC_INLN_DST_INLN:
if len(self.data) < 4:
raise dpkt.NeedData("for UDP ports")
self.src_port = struct.unpack("!H", self.data[0:2])[0]
self.dst_port = struct.unpack("!H", self.data[2:4])[0]
self.data = self.data[4:]
elif p == APv6Udp.HDR_PORTS_SRC_INLN_DST_F0XX:
if len(self.data) < 3:
raise dpkt.NeedData("for UDP ports")
self.src_port = struct.unpack("!H", self.data[0:2])[0]
self.dst_port = 0xf000 | self.data[2]
self.data = self.data[3:]
elif p == APv6Udp.HDR_PORTS_SRC_F0XX_DST_INLN:
if len(self.data) < 3:
raise dpkt.NeedData("for UDP ports")
self.src_port = 0xf000 | self.data[0]
self.dst_port = struct.unpack("!H", self.data[1:3])[0]
self.data = self.data[3:]
elif p == APv6Udp.HDR_PORTS_SRC_F0BX_DST_F0BX:
if len(self.data) < 1:
raise dpkt.NeedData("for UDP ports")
d = self.data[0]
self.src_port = 0xf0b0 | ((d >> 4) & 0b1111)
self.dst_port = 0xf0b0 | (d & 0b1111)
self.data = self.data[1:]
def parse_headers(f):
"""Return dict of HTTP headers parsed from a file object."""
d = {}
while 1:
line = f.readline()
# regular dpkt checks for premature end of headers
# but that's too picky
line = line.strip()
if not line:
break
l = line.split(None, 1)
if not l[0].endswith(':'):
raise dpkt.UnpackError('invalid header: %r' % line)
k = l[0][:-1].lower()
v = len(l) != 1 and l[1] or ''
if k in d:
d[k] += ','+v
else:
d[k] = v
return d
def _do_unpack_options(self, buf, oo=None):
self.opts = []
self.data = ''
oo = oo or self.__hdr_len__ - 4 # options offset
ol = self.len - oo - 4 # length
opts_buf = buf[oo:oo + ol]
while opts_buf:
opt = (PcapngOptionLE(opts_buf)
if self.__hdr_fmt__[0] == '<' else PcapngOption(opts_buf))
self.opts.append(opt)
opts_buf = opts_buf[len(opt):]
if opt.code == PCAPNG_OPT_ENDOFOPT:
break
# duplicate total length field
self._len = struct_unpack(self.__hdr_fmt__[0] + 'I', buf[-4:])[0]
if self._len != self.len:
raise dpkt.UnpackError('length fields do not match')
def unpack(self, buf):
f = cStringIO.StringIO(buf)
line = f.readline()
l = line.strip().split(None, 2)
if len(l) < 2 or not l[0].startswith(self.__proto) or not l[1].isdigit():
raise dpkt.UnpackError('invalid response: %r' % line)
self.version = l[0][len(self.__proto) + 1:]
self.status = l[1]
self.reason = l[2] if len(l) > 2 else ''
# RFC Sec 4.3.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3.
# For response messages, whether or not a message-body is included with
# a message is dependent on both the request method and the response
# status code (section 6.1.1). All responses to the HEAD request method
# MUST NOT include a message-body, even though the presence of entity-
# header fields might lead one to believe they do. All 1xx
# (informational), 204 (no content), and 304 (not modified) responses
# MUST NOT include a message-body. All other responses do include a
# message-body, although it MAY be of zero length.
is_body_allowed = int(self.status) >= 200 and 204 != int(self.status) != 304
Message.unpack(self, f.read(), is_body_allowed)
def parse_body(f, headers):
"""Return HTTP body parsed from a file object, given HTTP header dict."""
if headers.get('transfer-encoding', '').lower() == 'chunked':
l = []
found_end = False
while 1:
try:
sz = f.readline().split(None, 1)[0]
except IndexError:
raise dpkt.UnpackError('missing chunk size')
n = int(sz, 16)
if n == 0:
found_end = True
buf = f.read(n)
if f.readline().strip():
break
if n and len(buf) == n:
l.append(buf)
else:
break
if settings.strict_http_parse_body and not found_end:
raise dpkt.NeedData('premature end of chunked body')
body = ''.join(l)
elif 'content-length' in headers:
# Have observed malformed 0,0 content lengths
if headers['content-length'] == '0,0':
n = 0
else:
n = int(headers['content-length'])
body = f.read(n)
if len(body) != n:
logging.warn('HTTP content-length mismatch: expected %d, got %d',
n, len(body))
if settings.strict_http_parse_body:
raise dpkt.NeedData('short body (missing %d bytes)' %
(n - len(body)))
else:
# XXX - need to handle HTTP/0.9
body = ''
return body
def parse_headers(f):
"""Return dict of HTTP headers parsed from a file object."""
d = {}
while 1:
line = f.readline()
if not line:
raise dpkt.NeedData('premature end of headers')
line = line.strip()
if not line:
break
l = line.split(':', 1)
if len(l[0].split()) != 1:
raise dpkt.UnpackError('invalid header: %r' % line)
k = l[0].lower()
v = len(l) != 1 and l[1].lstrip() or ''
if k in d:
if not type(d[k]) is list:
d[k] = [d[k]]
d[k].append(v)
else:
d[k] = v
return d
def unpack_name(buf, off):
name = ''
saved_off = 0
for i in range(100): # XXX
n = ord(buf[off])
if n == 0:
off += 1
break
elif (n & 0xc0) == 0xc0:
ptr = struct.unpack('>H', buf[off:off + 2])[0] & 0x3fff
off += 2
if not saved_off:
saved_off = off
# XXX - don't use recursion!@#$
name = name + unpack_name(buf, ptr)[0] + '.'
break
else:
off += 1
name = name + buf[off:off + n] + '.'
if len(name) > 255:
raise dpkt.UnpackError('name longer than 255 bytes')
off += n
return name.strip('.'), off