def test_tlsmaster_client_random():
pcapfile = open("tests/pcaps/2019-05-01-airfrance-fr-traffic.pcap", "rb")
tlsmaster = "tests/tlsmasters/2019-05-01-airfrance-fr-tlsmaster.mitm"
if tlsmaster:
tls_master_secrets = read_tlsmaster(tlsmaster)
else:
tls_master_secrets = {}
handlers = {
25: smtp_handler,
80: http_handler,
8000: http_handler,
8080: http_handler,
443: lambda: https_handler(tls_master_secrets),
4443: lambda: https_handler(tls_master_secrets),
}
reader = PcapReader(pcapfile)
reader.tcp = TCPPacketStreamer(reader, handlers)
res = []
(s, ts, protocol, sent, recv) = next(reader.process())
assert sent.uri == "/"
assert sent.headers["host"] == "www.airfrance.fr"
assert recv.status == "200" and recv.reason == "OK"
assert recv.body[76:114] == b"09D925754D91E3F90CEE2306B9861683.a63s1"
assert recv.headers["content-type"] == "text/html;charset=UTF-8"
def test_unknownDatalinkException():
r = PcapReader(io.BytesIO(
struct.pack(
"IHHIIII", dpkt.pcap.TCPDUMP_MAGIC, dpkt.pcap.PCAP_VERSION_MAJOR,
dpkt.pcap.PCAP_VERSION_MINOR, 0, 0, 1500, 0
) + b"A"*16
))
with pytest.raises(UnknownDatalink):
list(r.process())
def test_pcap(self):
with open(os.path.join("tests", "pcaps", self.pcapfile), "rb") as f:
reader = PcapReader(f)
reader.tcp = TCPPacketStreamer(reader, self.handlers)
reader.raise_exceptions = self.use_exceptions
output = [self.format(*stream) for stream in reader.process()]
assert self.expected_output == output
def get_output(self, pcap):
reader = PcapReader(pcap)
reader.raise_exceptions = self.use_exceptions
list(reader.process())
key, exception = reader.exceptions.popitem()
output = [
exception["exception"],
os.path.basename(os.path.normpath(exception["trace"][-1][0]))
]
return output
def httpreplay(pcapfile, tlsmaster):
if tlsmaster:
tls_master_secrets = read_tlsmaster(tlsmaster)
else:
tls_master_secrets = {}
handlers = {
25: smtp_handler,
80: http_handler,
8000: http_handler,
8080: http_handler,
443: lambda: https_handler(tls_master_secrets),
4443: lambda: https_handler(tls_master_secrets),
}
reader = PcapReader(pcapfile)
reader.tcp = TCPPacketStreamer(reader, handlers)
for s, ts, protocol, sent, recv in reader.process():
print s, "%f" % ts, protocol, getattr(sent, "uri", None)
def httpreplay(pcapfile, tlsmaster):
if tlsmaster:
tls_master_secrets = read_tlsmaster(tlsmaster)
else:
tls_master_secrets = {}
handlers = {
25: smtp_handler,
80: http_handler,
8000: http_handler,
8080: http_handler,
443: lambda: https_handler(tls_master_secrets),
4443: lambda: https_handler(tls_master_secrets),
}
reader = PcapReader(pcapfile)
reader.tcp = TCPPacketStreamer(reader, handlers)
for s, ts, protocol, sent, recv in reader.process():
print(s, "%f" % ts, protocol, getattr(sent, "uri", None))
class SmtpTest(object):
handlers = {
25: smtp_handler,
587: smtp_handler,
}
def __init__(self, pcapfile):
self.reader = PcapReader(open(pcapfile, "rb"))
self.reader.tcp = TCPPacketStreamer(self.reader, self.handlers)
def get_sent_recv(self):
s, r = None, None
for s, ts, prot, sent, recv in self.reader.process():
s = sent
r = recv
return s, r
def test_pcap(self):
with open(os.path.join("tests", "pcaps", self.pcapfile), "rb") as f:
reader = PcapReader(f)
reader.tcp = TCPPacketStreamer(reader, self.handlers)
reader.tlsinfo = self.tlsinfo
reader.raise_exceptions = self.use_exceptions
output = [
self.format(*stream) for stream in reader.process()
]
assert len(self.expected_output) == len(output) and sorted(self.expected_output,key=lambda x: "" if x is None else str(x)) == sorted(output,key=lambda x: "" if x is None else str(x))
def pcap2mitm(pcapfile, mitmfile, tlsmaster, stream):
try:
from mitmproxy import models
from mitmproxy.flow import FlowWriter
from netlib.http import http1
from netlib.exceptions import HttpException
except ImportError:
raise click.Abort(
"In order to use this utility it is required to have the "
"mitmproxy tool installed (`pip install httpreplay[mitmproxy]`)"
)
class NetlibHttpProtocol(Protocol):
"""
Like HttpProtocol, but actually covering edge-cases.
"""
@staticmethod
def read_body(io, expected_size):
"""
Read a (malformed) HTTP body.
Returns:
A (body: bytes, is_malformed: bool) tuple.
"""
body_start = io.tell()
try:
content = b"".join(http1.read_body(io, expected_size, None))
if io.read(): # leftover?
raise HttpException()
return content, False
except HttpException:
io.seek(body_start)
return io.read(), True
def parse_request(self, ts, sent):
try:
sent = BytesIO(sent)
request = http1.read_request_head(sent)
body_size = http1.expected_http_body_size(request)
request.data.content, malformed = self.read_body(sent, body_size)
if malformed:
request.headers["X-Mitmproxy-Malformed-Body"] = "1"
return request
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def parse_response(self, ts, recv, request):
try:
recv = BytesIO(recv)
response = http1.read_response_head(recv)
body_size = http1.expected_http_body_size(request, response)
response.data.content, malformed = self.read_body(recv, body_size)
if malformed:
response.headers["X-Mitmproxy-Malformed-Body"] = "1"
return response
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def handle(self, s, ts, protocol, sent, recv):
if protocol not in ("tcp", "tls"):
self.parent.handle(s, ts, protocol, sent, recv)
return
req = None
if sent:
req = self.parse_request(ts, sent)
protocols = {
"tcp": "http",
"tls": "https",
}
# Only try to decode the HTTP response if the request was valid HTTP.
if req:
res = self.parse_response(ts, recv, req)
# Report this stream as being a valid HTTP stream.
self.parent.handle(s, ts, protocols[protocol],
req or sent, res)
else:
# This wasn't a valid HTTP stream so we forward the original TCP
# or TLS stream straight ahead to our parent.
self.parent.handle(s, ts, protocol, sent, recv)
if tlsmaster:
tlsmaster = read_tlsmaster(tlsmaster)
else:
tlsmaster = {}
netlib_http_handler = lambda: NetlibHttpProtocol()
netlib_https_handler = lambda: TLSStream(NetlibHttpProtocol(), tlsmaster)
handlers = {
443: netlib_https_handler,
4443: netlib_https_handler,
'generic': netlib_http_handler,
}
reader = PcapReader(pcapfile)
reader.tcp = TCPPacketStreamer(reader, handlers)
writer = FlowWriter(mitmfile)
l = reader.process()
if not stream:
# Sort the http/https requests and responses by their timestamp.
l = sorted(l, key=lambda x: x[1])
for addrs, timestamp, protocol, sent, recv in l:
if protocol not in ("http", "https"):
continue
srcip, srcport, dstip, dstport = addrs
client_conn = models.ClientConnection.make_dummy((srcip, srcport))
client_conn.timestamp_start = timestamp
server_conn = models.ServerConnection.make_dummy((dstip, dstport))
server_conn.timestamp_start = timestamp
flow = models.HTTPFlow(client_conn, server_conn)
flow.request = models.HTTPRequest.wrap(sent)
flow.request.host, flow.request.port = dstip, dstport
flow.request.scheme = protocol
if recv:
flow.response = models.HTTPResponse.wrap(recv)
writer.add(flow)
def test_init_reader():
a = PcapReader("tests/pcaps/test.pcap")
b = PcapReader(open("tests/pcaps/test.pcap", "rb"))
assert list(a.pcap) == list(b.pcap)
def pcap2mitm(pcapfile, mitmfile, tlsmaster=None, stream=False):
try:
from mitmproxy import models
from mitmproxy.flow import FlowWriter
from netlib.exceptions import HttpException
from netlib.http import http1
except ImportError:
log.warning(
"In order to use this utility it is required to have the "
"mitmproxy tool installed (`pip install httpreplay[mitmproxy]`)")
return False
if tlsmaster:
tlsmaster = read_tlsmaster(tlsmaster)
else:
tlsmaster = {}
handlers = {
443: lambda: https_handler(tlsmaster),
4443: lambda: https_handler(tlsmaster),
"generic": http_handler,
}
reader = PcapReader(pcapfile)
reader.tcp = TCPPacketStreamer(reader, handlers)
writer = FlowWriter(mitmfile)
l = reader.process()
if not stream:
# Sort the http/https requests and responses by their timestamp.
l = sorted(l, key=lambda x: x[1])
for s, ts, protocol, sent, recv in l:
if protocol not in ("http", "https"):
continue
srcip, srcport, dstip, dstport = s
client_conn = models.ClientConnection.make_dummy((srcip, srcport))
client_conn.timestamp_start = ts
server_conn = models.ServerConnection.make_dummy((dstip, dstport))
server_conn.timestamp_start = ts
flow = models.HTTPFlow(client_conn, server_conn)
try:
sent = io.BytesIO(sent.raw)
request = http1.read_request_head(sent)
body_size = http1.expected_http_body_size(request)
request.data.content = "".join(
http1.read_body(sent, body_size, None))
except HttpException as e:
log.warning("Error parsing HTTP request: %s", e)
continue
flow.request = models.HTTPRequest.wrap(request)
flow.request.timestamp_start = client_conn.timestamp_start
flow.request.host = dstip
flow.request.port = dstport
flow.request.scheme = protocol
try:
recv = io.BytesIO(recv.raw)
response = http1.read_response_head(recv)
body_size = http1.expected_http_body_size(request, response)
response.data.content = "".join(
http1.read_body(recv, body_size, None))
except HttpException as e:
log.warning("Error parsing HTTP response: %s", e)
# Fall through (?)
flow.response = models.HTTPResponse.wrap(response)
flow.response.timestamp_start = server_conn.timestamp_start
flow.id = str(
uuid.UUID(bytes=hashlib.md5(
b"%d%d%s%s" %
(client_conn.timestamp_start, server_conn.timestamp_start,
request.data.content, response.data.content)).digest()))
writer.add(flow)
return True
def process(self, flowmapping, ports=None):
"""Reads a PCAP and adds the reqeusts that match the flow mapping
to a dictionary of reports it will return. Reports will contain
all found requests made for a url
@param flowmapping: a dictionary of netflow:url values"""
tlspath = cwd("tlsmaster.txt", analysis=self.task_id)
tlsmaster = {}
if os.path.exists(tlspath):
tlsmaster = read_tlsmaster(tlspath)
if tlsmaster or not self.handlers:
self._create_handlers(tlsmaster, ports=ports)
pcap_fp = open(cwd("dump.pcap", analysis=self.task_id))
reader = PcapReader(pcap_fp)
if self.offset and self.offset > 0:
pcap_fp.seek(self.offset)
reader.set_tcp_handler(TCPPacketStreamer(reader, self.handlers))
reports = {}
try:
for flow, timestamp, protocol, sent, recv in reader.process():
tracked_url = flowmapping.get(flow)
if not tracked_url:
continue
if not sent and not recv:
continue
if isinstance(sent, dpkt.http.Request):
url = "%s://%s%s" % (protocol,
sent.headers.get(
"host", "%s:%s" %
(flow[2], flow[3])), sent.uri)
else:
url = "%s://%s:%s" % (protocol, flow[2], flow[3])
report = reports.setdefault(tracked_url, {})
requested = report.setdefault("requested", [])
logs = report.setdefault("log", {})
if url not in requested:
requested.append(url)
if not isinstance(recv, (dpkt.http.Response, basestring)):
recv = recv.raw
requestlog = logs.setdefault(url, [])
requestlog.append({
"time":
timestamp,
"request":
bytes(sent)[:self.MAX_REQUEST_SIZE],
"response":
bytes(recv)[:self.MAX_REQUEST_SIZE]
})
return reports
except Exception as e:
log.exception("Failure while extracting requests from PCAP")
return reports
finally:
self.offset = pcap_fp.tell()
pcap_fp.close()
def __init__(self, pcapfile):
self.reader = PcapReader(open(pcapfile, "rb"))
self.reader.tcp = TCPPacketStreamer(self.reader, self.handlers)
def pcap2mitm(pcapfile, mitmfile, tlsmaster, stream):
try:
from mitmproxy import models
from mitmproxy.flow import FlowWriter
from netlib.http import http1
from netlib.exceptions import HttpException
except ImportError:
raise click.Abort(
"In order to use this utility it is required to have the "
"mitmproxy tool installed (`pip install httpreplay[mitmproxy]`)")
class NetlibHttpProtocol(Protocol):
"""
Like HttpProtocol, but actually covering edge-cases.
"""
@staticmethod
def read_body(io, expected_size):
"""
Read a (malformed) HTTP body.
Returns:
A (body: bytes, is_malformed: bool) tuple.
"""
body_start = io.tell()
try:
content = b"".join(http1.read_body(io, expected_size, None))
if io.read(): # leftover?
raise HttpException()
return content, False
except HttpException:
io.seek(body_start)
return io.read(), True
def parse_request(self, ts, sent):
try:
sent = BytesIO(sent)
request = http1.read_request_head(sent)
body_size = http1.expected_http_body_size(request)
request.data.content, malformed = self.read_body(
sent, body_size)
if malformed:
request.headers["X-Mitmproxy-Malformed-Body"] = "1"
return request
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def parse_response(self, ts, recv, request):
try:
recv = BytesIO(recv)
response = http1.read_response_head(recv)
body_size = http1.expected_http_body_size(request, response)
response.data.content, malformed = self.read_body(
recv, body_size)
if malformed:
response.headers["X-Mitmproxy-Malformed-Body"] = "1"
return response
except HttpException as e:
log.warning("{!r} (timestamp: {})".format(e, ts))
def handle(self, s, ts, protocol, sent, recv):
if protocol not in ("tcp", "tls"):
self.parent.handle(s, ts, protocol, sent, recv)
return
req = None
if sent:
req = self.parse_request(ts, sent)
protocols = {
"tcp": "http",
"tls": "https",
}
# Only try to decode the HTTP response if the request was valid HTTP.
if req:
res = self.parse_response(ts, recv, req)
# Report this stream as being a valid HTTP stream.
self.parent.handle(s, ts, protocols[protocol], req or sent,
res)
else:
# This wasn't a valid HTTP stream so we forward the original TCP
# or TLS stream straight ahead to our parent.
self.parent.handle(s, ts, protocol, sent, recv)
if tlsmaster:
tlsmaster = read_tlsmaster(tlsmaster)
else:
tlsmaster = {}
netlib_http_handler = lambda: NetlibHttpProtocol()
netlib_https_handler = lambda: TLSStream(NetlibHttpProtocol(), tlsmaster)
handlers = {
443: netlib_https_handler,
4443: netlib_https_handler,
"generic": netlib_http_handler,
}
reader = PcapReader(pcapfile)
reader.tcp = TCPPacketStreamer(reader, handlers)
writer = FlowWriter(mitmfile)
l = reader.process()
if not stream:
# Sort the http/https requests and responses by their timestamp.
l = sorted(l, key=lambda x: x[1])
for addrs, timestamp, protocol, sent, recv in l:
if protocol not in ("http", "https"):
continue
srcip, srcport, dstip, dstport = addrs
client_conn = models.ClientConnection.make_dummy((srcip, srcport))
client_conn.timestamp_start = timestamp
server_conn = models.ServerConnection.make_dummy((dstip, dstport))
server_conn.timestamp_start = timestamp
flow = models.HTTPFlow(client_conn, server_conn)
flow.request = models.HTTPRequest.wrap(sent)
flow.request.host, flow.request.port = dstip, dstport
flow.request.scheme = protocol
if recv:
flow.response = models.HTTPResponse.wrap(recv)
writer.add(flow)
def test_unknownIpProtocolException():
r = PcapReader(open(
os.path.join("tests", "pcaps", "unknownIpProtocol.pcap"), "rb"
))
with pytest.raises(UnknownIpProtocol):
list(r.process())
