def load_flows(path):
flows = []
with open(path, 'rb') as f:
flows.extend(io.FlowReader(f).stream())
return flows
def test_load(tdata, dumpfile, url, count):
with open(tdata.path("mitmproxy/data/" + dumpfile), "rb") as f:
flow_reader = io.FlowReader(f)
flows = list(flow_reader.stream())
assert len(flows) == count
assert flows[-1].request.url.startswith(url)
def test_cannot_convert(tdata):
with open(tdata.path("mitmproxy/data/dumpfile-010.mitm"), "rb") as f:
flow_reader = io.FlowReader(f)
with pytest.raises(exceptions.FlowReadException):
list(flow_reader.stream())
def post(self):
self.view.clear()
bio = BytesIO(self.filecontents)
self.master.load_flows(io.FlowReader(bio))
bio.close()
def post(self):
self.view.clear()
bio = BytesIO(self.filecontents)
for i in io.FlowReader(bio).stream():
self.master.load_flow(i)
bio.close()
#!/usr/bin/env python
"""
Read a mitmproxy dump file.
"""
from mitmproxy import io, http
from mitmproxy.exceptions import FlowReadException
import pprint
import sys
with open(sys.argv[1], "rb") as logfile:
freader = io.FlowReader(logfile)
pp = pprint.PrettyPrinter(indent=4)
try:
for f in freader.stream():
print(f)
if isinstance(f, http.HTTPFlow):
print(f.request.host)
pp.pprint(f.get_state())
print("")
except FlowReadException as e:
print(f"Flow file corrupted: {e}")
def rd(p):
x = io.FlowReader(open(p, "rb"))
return list(x.stream())
def rd(p):
with open(p, "rb") as f:
x = io.FlowReader(f)
return list(x.stream())
def process_activity(self, mitmlog):
"""Collect tail of the activity log and turn into an activity record.
Args:
mitmlog(file): open file object
Returns:
ar(): activity record
"""
# get an fstream generator object
fstream = mitmio.FlowReader(mitmlog).stream()
# no context yet. not a notebook or console
self.is_console = False
self.is_notebook = False
while True:
try:
f = next(fstream)
except StopIteration:
break
except FlowReadException as e:
logger.info("MITM Flow file corrupted: {}. Exiting.".format(e))
break
st: Dict = f.get_state()
is_png = False
is_gzip = False
is_json = False
# Check response types for images and json
for header in st['response']['headers']:
# png images
if header[0] == b'Content-Type':
if header[1] == b'image/png':
is_png = True
# json
if header[0] == b'Content-Type':
if header[1] == b'application/json':
is_json = True
if header[0] == b'Content-Encoding':
if header[1] == b'gzip':
is_gzip = True
else:
# Not currently used, but useful for debugging and potentially in future
encoding = header[1]
# process images
if is_png:
if is_gzip:
self._parse_image(st)
else:
logger.error(
f"RSERVER Found image/png that was not gzip encoded.")
if is_json:
self._parse_json(st, is_gzip)
# Flush cell data IFF anything happened
if self.current_cell.code:
self.cell_data.append(self.current_cell)
self.store_record()
self.current_cell = ExecutionData()
self.cell_data = list()
self.chunk_id = None
def searchmitmflow(flowfile, searchterm):
global found, foundunprot, foundprot
# Searches are case insensitive
pcre = re.compile('(?i)' + searchterm)
# Create a dictionary of all of the messages in the flow
with open(flowfile, 'rb') as logfile:
fr = io.FlowReader(logfile)
msgnum = 0
flowdict = {}
messages = []
for msg in fr.stream():
messagedict = {}
responsedict = {}
requestdict = {}
messagedict['msgnum'] = msgnum
requestdict['uri'] = msg.request.pretty_url
requestdict['method'] = msg.request.method
requestdict['scheme'] = msg.request.scheme
requestdict['headers'] = msg.request.headers
requestcontent = msg.request.content
decodedcontent = decodecontent(requestcontent)
if decodedcontent:
# mitmproxy found a way to decode the content
requestdict['content'] = decodedcontent
else:
# just take the raw bytes
requestdict['content'] = requestcontent
try:
responsedict['headers'] = msg.response.headers
except AttributeError:
responsedict['headers'] = ''
try:
responsecontent = msg.response.content
except AttributeError:
responsecontent = ''
try:
decodedcontent = decodecontent(responsecontent)
if decodedcontent:
# mitmproxy found a way to decode the content
responsedict['content'] = decodecontent(responsecontent)
else:
# just take the raw bytes
responsedict['content'] = responsecontent
except AttributeError:
responsedict['content'] = ''
messagedict['request'] = requestdict
messagedict['response'] = responsedict
# print(messagedict)
messages.append(messagedict)
msgnum = msgnum + 1
flowdict['messages'] = messages
# Check for matches in the flow dictionary
for message in flowdict['messages']:
msgnum = message['msgnum']
for key in message:
if key == 'msgnum':
continue
else:
for value in message[key].values():
if pcre.search(str(value)):
found = True
print(
color.bright(
color.green(
'Found match for %s in %s message [%s] field [%s]:'
%
(searchterm, flowfile, msgnum + 1, key))))
if message['request']['scheme'] == 'https':
if flowfile.endswith('ssltest.log'):
foundunprot = True
print(
color.bright(
color.red(
'%s found in non-validated HTTPS traffic!'
% searchterm)))
elif flowfile.endswith('flows.log'):
foundprot = True
color.bright(
'%s found in validated HTTPS traffic' %
searchterm)
elif message['request']['scheme'] == 'http':
foundunenc = True
print(
color.bright(
color.red(
'%s found in unencrypted HTTP traffic!'
% searchterm)))
print(str(value))
def test_load():
with open(tutils.test_data.path("data/dumpfile-011"), "rb") as f:
flow_reader = io.FlowReader(f)
flows = list(flow_reader.stream())
assert len(flows) == 1
assert flows[0].request.url == "https://example.com/"
def test_cannot_convert():
with open(tutils.test_data.path("data/dumpfile-010"), "rb") as f:
flow_reader = io.FlowReader(f)
with tutils.raises(exceptions.FlowReadException):
list(flow_reader.stream())
