def test_binaryops(self):
a = filt.parse("~u foobar | ~h voing")
isinstance(a, filt.FOr)
self._dump(a)
a = filt.parse("~u foobar & ~h voing")
isinstance(a, filt.FAnd)
self._dump(a)
def test_quoting(self):
a = filt.parse("~u 'foo ~u bar' ~u voing")
assert a.lst[0].expr == "foo ~u bar"
assert a.lst[1].expr == "voing"
self._dump(a)
a = filt.parse("~u foobar")
assert a.expr == "foobar"
a = filt.parse(r"~u 'foobar\"\''")
assert a.expr == "foobar\"'"
a = filt.parse(r'~u "foo \'bar"')
assert a.expr == "foo 'bar"
def __init__(self, server, options):
flow.FlowMaster.__init__(self, options, server, flow.State())
self.addons.add(*builtins.default_addons())
# This line is just for type hinting
self.options = self.options # type: Options
self.o = options
self.showhost = options.showhost
self.replay_ignore_params = options.replay_ignore_params
self.replay_ignore_content = options.replay_ignore_content
self.replay_ignore_host = options.replay_ignore_host
self.refresh_server_playback = options.refresh_server_playback
self.replay_ignore_payload_params = options.replay_ignore_payload_params
self.set_stream_large_bodies(options.stream_large_bodies)
if self.server and self.server.config.http2 and not tcp.HAS_ALPN: # pragma: no cover
print("ALPN support missing (OpenSSL 1.0.2+ required)!\n"
"HTTP/2 is disabled. Use --no-http2 to silence this warning.",
file=sys.stderr)
if options.filtstr:
self.filt = filt.parse(options.filtstr)
else:
self.filt = None
if options.replacements:
for i in options.replacements:
self.replacehooks.add(*i)
if options.setheaders:
for i in options.setheaders:
self.setheaders.add(*i)
if options.server_replay:
self.start_server_playback(
self._readflow(options.server_replay),
options.kill, options.rheaders,
not options.keepserving,
options.nopop,
options.replay_ignore_params,
options.replay_ignore_content,
options.replay_ignore_payload_params,
options.replay_ignore_host
)
if options.client_replay:
self.start_client_playback(
self._readflow(options.client_replay),
not options.keepserving
)
if options.rfile:
try:
self.load_flows_file(options.rfile)
except exceptions.FlowReadException as v:
self.add_event("Flow file corrupted.", "error")
raise DumpError(v)
if self.options.app:
self.start_app(self.options.app_host, self.options.app_port)
def configure(self, options):
if options.stickyauth:
flt = filt.parse(options.stickyauth)
if not flt:
raise exceptions.OptionsError(
"stickyauth: invalid filter expression: %s" % options.stickyauth
)
self.flt = flt
def configure(self, options, updated):
if options.stickyauth:
flt = filt.parse(options.stickyauth)
if not flt:
raise exceptions.OptionsError(
"stickyauth: invalid filter expression: %s" % options.stickyauth
)
self.flt = flt
def set_intercept(self, txt):
if txt:
f = filt.parse(txt)
if not f:
return "Invalid filter expression."
self.intercept = f
else:
self.intercept = None
def set_intercept(self, txt):
if txt:
f = filt.parse(txt)
if not f:
return "Invalid filter expression."
self.intercept = f
else:
self.intercept = None
def configure(self, options, updated):
if options.stickycookie:
flt = filt.parse(options.stickycookie)
if not flt:
raise exceptions.OptionsError(
"stickycookie: invalid filter expression: %s" % options.stickycookie
)
self.flt = flt
def is_error(self, col, val):
if col == 0:
if not filt.parse(val):
return "Invalid filter specification."
elif col == 1:
try:
re.compile(val)
except re.error:
return "Invalid regular expression."
return False
def test_handle_response(self):
s = flow.StickyAuthState(filt.parse(".*"))
f = tutils.tflow(resp=True)
f.request.headers["authorization"] = "foo"
s.handle_request(f)
assert "address" in s.hosts
f = tutils.tflow(resp=True)
s.handle_request(f)
assert f.request.headers["authorization"] == "foo"
def is_error(self, col, val):
if col == 0:
if not filt.parse(val):
return "Invalid filter specification."
elif col == 1:
try:
re.compile(val)
except re.error:
return "Invalid regular expression."
return False
def test_handle_response(self):
s = flow.StickyAuthState(filt.parse(".*"))
f = tutils.tflow(resp=True)
f.request.headers["authorization"] = "foo"
s.handle_request(f)
assert "address" in s.hosts
f = tutils.tflow(resp=True)
s.handle_request(f)
assert f.request.headers["authorization"] == "foo"
def set_stickyauth(self, txt):
if txt:
flt = filt.parse(txt)
if not flt:
return "Invalid filter expression."
self.stickyauth_state = modules.StickyAuthState(flt)
self.stickyauth_txt = txt
else:
self.stickyauth_state = None
self.stickyauth_txt = None
def configure(self, options, updated):
if options.filtstr:
self.filter = filt.parse(options.filtstr)
if not self.filter:
raise exceptions.OptionsError("Invalid filter expression: %s" %
options.filtstr)
else:
self.filter = None
self.flow_detail = options.flow_detail
self.outfp = options.tfile
self.showhost = options.showhost
def set_view_filter(self, txt):
if txt == self.filter_txt:
return
if txt:
f = filt.parse(txt)
if not f:
return "Invalid filter expression."
self.view._close()
self.view = FlowView(self.flows, f)
else:
self.view._close()
self.view = FlowView(self.flows, None)
def test_simple(self):
assert not filt.parse("~b")
assert filt.parse("~q")
assert filt.parse("~c 10")
assert filt.parse("~m foobar")
assert filt.parse("~u foobar")
assert filt.parse("~q ~c 10")
p = filt.parse("~q ~c 10")
self._dump(p)
assert len(p.lst) == 2
def set_limit(self, txt):
if txt == self.limit_txt:
return
if txt:
f = filt.parse(txt)
if not f:
return "Invalid filter expression."
self.view._close()
self.view = FlowView(self.flows, f)
else:
self.view._close()
self.view = FlowView(self.flows, None)
def configure(self, options, updated):
if options.filtstr:
self.filter = filt.parse(options.filtstr)
if not self.filter:
raise exceptions.OptionsError(
"Invalid filter expression: %s" % options.filtstr
)
else:
self.filter = None
self.flow_detail = options.flow_detail
self.outfp = options.tfile
self.showhost = options.showhost
def configure(self, options, updated):
"""
options.setheaders is a tuple of (fpatt, header, value)
fpatt: String specifying a filter pattern.
header: Header name.
value: Header value string
"""
for fpatt, header, value in options.setheaders:
cpatt = filt.parse(fpatt)
if not cpatt:
raise exceptions.OptionsError(
"Invalid setheader filter pattern %s" % fpatt
)
self.lst.append((fpatt, header, value, cpatt))
def configure(self, options, updated):
"""
options.setheaders is a tuple of (fpatt, header, value)
fpatt: String specifying a filter pattern.
header: Header name.
value: Header value string
"""
for fpatt, header, value in options.setheaders:
cpatt = filt.parse(fpatt)
if not cpatt:
raise exceptions.OptionsError(
"Invalid setheader filter pattern %s" % fpatt
)
self.lst.append((fpatt, header, value, cpatt))
def test_response(self):
c = (
"SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; "
"Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; "
)
s, f = self._response(c, "host")
assert not s.jar.keys()
s, f = self._response(c, "www.google.com")
assert list(s.jar.keys())[0] == ('.google.com', 80, '/')
s, f = self._response("SSID=mooo", "www.google.com")
assert list(s.jar.keys())[0] == ('www.google.com', 80, '/')
# Test setting of multiple cookies
c1 = "somecookie=test; Path=/"
c2 = "othercookie=helloworld; Path=/"
s, f = self._response(c1, "www.google.com")
f.response.headers["Set-Cookie"] = c2
s.handle_response(f)
googlekey = list(s.jar.keys())[0]
assert len(s.jar[googlekey].keys()) == 2
# Test setting of weird cookie keys
s = flow.StickyCookieState(filt.parse(".*"))
f = tutils.tflow(req=netlib.tutils.treq(host="www.google.com", port=80), resp=True)
cs = [
"foo/bar=hello",
"foo:bar=world",
"foo@bar=fizz",
"foo,bar=buzz",
]
for c in cs:
f.response.headers["Set-Cookie"] = c
s.handle_response(f)
googlekey = list(s.jar.keys())[0]
assert len(s.jar[googlekey]) == len(cs)
# Test overwriting of a cookie value
c1 = "somecookie=helloworld; Path=/"
c2 = "somecookie=newvalue; Path=/"
s, f = self._response(c1, "www.google.com")
f.response.headers["Set-Cookie"] = c2
s.handle_response(f)
googlekey = list(s.jar.keys())[0]
assert len(s.jar[googlekey]) == 1
assert list(s.jar[googlekey]["somecookie"].values())[0] == "newvalue"
def test_response(self):
c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \
"Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; "
s, f = self._response(c, "host")
assert not s.jar.keys()
s, f = self._response(c, "www.google.com")
assert s.jar.keys()
s, f = self._response("SSID=mooo", "www.google.com")
assert s.jar.keys()[0] == ('www.google.com', 80, '/')
# Test setting of multiple cookies
c1 = "somecookie=test; Path=/"
c2 = "othercookie=helloworld; Path=/"
s, f = self._response(c1, "www.google.com")
f.response.headers["Set-Cookie"] = c2
s.handle_response(f)
googlekey = s.jar.keys()[0]
assert len(s.jar[googlekey].keys()) == 2
# Test setting of weird cookie keys
s = flow.StickyCookieState(filt.parse(".*"))
f = tutils.tflow(req=netlib.tutils.treq(host="www.google.com",
port=80),
resp=True)
cs = [
"foo/bar=hello",
"foo:bar=world",
"foo@bar=fizz",
"foo,bar=buzz",
]
for c in cs:
f.response.headers["Set-Cookie"] = c
s.handle_response(f)
googlekey = s.jar.keys()[0]
assert len(s.jar[googlekey].keys()) == len(cs)
# Test overwriting of a cookie value
c1 = "somecookie=helloworld; Path=/"
c2 = "somecookie=newvalue; Path=/"
s, f = self._response(c1, "www.google.com")
f.response.headers["Set-Cookie"] = c2
s.handle_response(f)
googlekey = s.jar.keys()[0]
assert len(s.jar[googlekey].keys()) == 1
assert s.jar[googlekey]["somecookie"].items()[0][1] == "newvalue"
def add(self, fpatt, header, value):
"""
Add a set header hook.
fpatt: String specifying a filter pattern.
header: Header name.
value: Header value string
Returns True if hook was added, False if the pattern could not be
parsed.
"""
cpatt = filt.parse(fpatt)
if not cpatt:
return False
self.lst.append((fpatt, header, value, cpatt))
return True
def add(self, fpatt, header, value):
"""
Add a set header hook.
fpatt: String specifying a filter pattern.
header: Header name.
value: Header value string
Returns True if hook was added, False if the pattern could not be
parsed.
"""
cpatt = filt.parse(fpatt)
if not cpatt:
return False
self.lst.append((fpatt, header, value, cpatt))
return True
def test_filter(self):
sio = StringIO()
fl = filt.parse("~c 200")
w = flow.FilteredFlowWriter(sio, fl)
f = tutils.tflow(resp=True)
f.response.status_code = 200
w.add(f)
f = tutils.tflow(resp=True)
f.response.status_code = 201
w.add(f)
sio.seek(0)
r = flow.FlowReader(sio)
assert len(list(r.stream()))
def test_filter(self):
sio = StringIO()
fl = filt.parse("~c 200")
w = flow.FilteredFlowWriter(sio, fl)
f = tutils.tflow(resp=True)
f.response.status_code = 200
w.add(f)
f = tutils.tflow(resp=True)
f.response.status_code = 201
w.add(f)
sio.seek(0)
r = flow.FlowReader(sio)
assert len(list(r.stream()))
def _parse_hook(s):
sep, rem = s[0], s[1:]
parts = rem.split(sep, 2)
if len(parts) == 2:
patt = ".*"
a, b = parts
elif len(parts) == 3:
patt, a, b = parts
else:
raise ParseException("Malformed hook specifier - too few clauses: %s" %
s)
if not a:
raise ParseException("Empty clause: %s" % str(patt))
if not filt.parse(patt):
raise ParseException("Malformed filter pattern: %s" % patt)
return patt, a, b
def add(self, fpatt, rex, s):
"""
add a replacement hook.
fpatt: a string specifying a filter pattern.
rex: a regular expression.
s: the replacement string
returns true if hook was added, false if the pattern could not be
parsed.
"""
cpatt = filt.parse(fpatt)
if not cpatt:
return False
try:
re.compile(rex)
except re.error:
return False
self.lst.append((fpatt, rex, s, cpatt))
return True
def add(self, fpatt, rex, s):
"""
add a replacement hook.
fpatt: a string specifying a filter pattern.
rex: a regular expression.
s: the replacement string
returns true if hook was added, false if the pattern could not be
parsed.
"""
cpatt = filt.parse(fpatt)
if not cpatt:
return False
try:
re.compile(rex)
except re.error:
return False
self.lst.append((fpatt, rex, s, cpatt))
return True
def _parse_hook(s):
sep, rem = s[0], s[1:]
parts = rem.split(sep, 2)
if len(parts) == 2:
patt = ".*"
a, b = parts
elif len(parts) == 3:
patt, a, b = parts
else:
raise ParseException(
"Malformed hook specifier - too few clauses: %s" % s
)
if not a:
raise ParseException("Empty clause: %s" % str(patt))
if not filt.parse(patt):
raise ParseException("Malformed filter pattern: %s" % patt)
return patt, a, b
def configure(self, options, updated):
"""
.replacements is a list of tuples (fpat, rex, s):
fpatt: a string specifying a filter pattern.
rex: a regular expression, as bytes.
s: the replacement string, as bytes
"""
lst = []
for fpatt, rex, s in options.replacements:
cpatt = filt.parse(fpatt)
if not cpatt:
raise exceptions.OptionsError("Invalid filter pattern: %s" %
fpatt)
try:
re.compile(rex)
except re.error as e:
raise exceptions.OptionsError(
"Invalid regular expression: %s - %s" % (rex, str(e)))
lst.append((rex, s, cpatt))
self.lst = lst
def configure(self, options):
"""
.replacements is a list of tuples (fpat, rex, s):
fpatt: a string specifying a filter pattern.
rex: a regular expression.
s: the replacement string
"""
lst = []
for fpatt, rex, s in options.replacements:
cpatt = filt.parse(fpatt)
if not cpatt:
raise exceptions.OptionsError(
"Invalid filter pattern: %s" % fpatt
)
try:
re.compile(rex)
except re.error as e:
raise exceptions.OptionsError(
"Invalid regular expression: %s - %s" % (rex, str(e))
)
lst.append((rex, s, cpatt))
self.lst = lst
def test_domain_match(self):
s = flow.StickyCookieState(filt.parse(".*"))
assert s.domain_match("www.google.com", ".google.com")
assert s.domain_match("google.com", ".google.com")
def _response(self, cookie, host):
s = flow.StickyCookieState(filt.parse(".*"))
f = tutils.tflow(req=netlib.tutils.treq(host=host, port=80), resp=True)
f.response.headers["Set-Cookie"] = cookie
s.handle_response(f)
return s, f
def is_error(self, col, val):
if col == 0:
if not filt.parse(val):
return "Invalid filter specification"
return False
def test_not(self):
a = filt.parse("!~h test")
assert a.itm.expr == "test"
a = filt.parse("!(~u test & ~h bar)")
assert a.itm.lst[0].expr == "test"
self._dump(a)
def test_parse_err(self):
assert filt.parse("~h [") is None
def __init__(self, spec):
self.filter = filt.parse(spec)
def test_pyparsing_bug(extract_tb):
"""https://github.com/mitmproxy/mitmproxy/issues/1087"""
# The text is a string with leading and trailing whitespace stripped; if the source is not available it is None.
extract_tb.return_value = [("", 1, "test", None)]
assert filt.parse("test")
def is_error(self, col, val):
if col == 0:
if not filt.parse(val):
return "Invalid filter specification"
return False
def q(self, q, o):
return filt.parse(q)(o)
def start():
if len(sys.argv) != 2:
raise ValueError("Usage: -s 'filt.py FILTER'")
state["filter"] = filt.parse(sys.argv[1])
def test_nesting(self):
a = filt.parse("(~u foobar & ~h voing)")
assert a.lst[0].expr == "foobar"
self._dump(a)
def _response(self, cookie, host):
s = flow.StickyCookieState(filt.parse(".*"))
f = tutils.tflow(req=netlib.tutils.treq(host=host, port=80), resp=True)
f.response.headers["Set-Cookie"] = cookie
s.handle_response(f)
return s, f
def test_naked_url(self):
a = filt.parse("foobar ~h rex")
assert a.lst[0].expr == "foobar"
assert a.lst[1].expr == "rex"
self._dump(a)
def test_domain_match(self):
s = flow.StickyCookieState(filt.parse(".*"))
assert s.domain_match("www.google.com", ".google.com")
assert s.domain_match("google.com", ".google.com")
def __init__(self, server, options, outfile=None):
flow.FlowMaster.__init__(self, server, flow.State())
self.outfile = outfile
self.o = options
self.anticache = options.anticache
self.anticomp = options.anticomp
self.showhost = options.showhost
self.replay_ignore_params = options.replay_ignore_params
self.replay_ignore_content = options.replay_ignore_content
self.replay_ignore_host = options.replay_ignore_host
self.refresh_server_playback = options.refresh_server_playback
self.replay_ignore_payload_params = options.replay_ignore_payload_params
self.set_stream_large_bodies(options.stream_large_bodies)
if self.server and self.server.config.http2 and not tcp.HAS_ALPN: # pragma: no cover
print(
"ALPN support missing (OpenSSL 1.0.2+ required)!\n"
"HTTP/2 is disabled. Use --no-http2 to silence this warning.",
file=sys.stderr)
if options.filtstr:
self.filt = filt.parse(options.filtstr)
else:
self.filt = None
if options.stickycookie:
self.set_stickycookie(options.stickycookie)
if options.stickyauth:
self.set_stickyauth(options.stickyauth)
if options.outfile:
err = self.start_stream_to_path(options.outfile[0],
options.outfile[1], self.filt)
if err:
raise DumpError(err)
if options.replacements:
for i in options.replacements:
self.replacehooks.add(*i)
if options.setheaders:
for i in options.setheaders:
self.setheaders.add(*i)
if options.server_replay:
self.start_server_playback(self._readflow(options.server_replay),
options.kill, options.rheaders,
not options.keepserving, options.nopop,
options.replay_ignore_params,
options.replay_ignore_content,
options.replay_ignore_payload_params,
options.replay_ignore_host)
if options.client_replay:
self.start_client_playback(self._readflow(options.client_replay),
not options.keepserving)
scripts = options.scripts or []
for command in scripts:
try:
self.load_script(command, use_reloader=True)
except exceptions.ScriptException as e:
raise DumpError(str(e))
if options.rfile:
try:
self.load_flows_file(options.rfile)
except exceptions.FlowReadException as v:
self.add_event("Flow file corrupted.", "error")
raise DumpError(v)
if self.o.app:
self.start_app(self.o.app_host, self.o.app_port)
def __init__(self, server, options, outfile=None):
flow.FlowMaster.__init__(self, server, flow.State())
self.outfile = outfile
self.o = options
self.anticache = options.anticache
self.anticomp = options.anticomp
self.showhost = options.showhost
self.replay_ignore_params = options.replay_ignore_params
self.replay_ignore_content = options.replay_ignore_content
self.replay_ignore_host = options.replay_ignore_host
self.refresh_server_playback = options.refresh_server_playback
self.replay_ignore_payload_params = options.replay_ignore_payload_params
self.set_stream_large_bodies(options.stream_large_bodies)
if self.server and self.server.config.http2 and not tcp.HAS_ALPN: # pragma: no cover
print("ALPN support missing (OpenSSL 1.0.2+ required)!\n"
"HTTP/2 is disabled. Use --no-http2 to silence this warning.",
file=sys.stderr)
if options.filtstr:
self.filt = filt.parse(options.filtstr)
else:
self.filt = None
if options.stickycookie:
self.set_stickycookie(options.stickycookie)
if options.stickyauth:
self.set_stickyauth(options.stickyauth)
if options.outfile:
err = self.start_stream_to_path(
options.outfile[0],
options.outfile[1],
self.filt
)
if err:
raise DumpError(err)
if options.replacements:
for i in options.replacements:
self.replacehooks.add(*i)
if options.setheaders:
for i in options.setheaders:
self.setheaders.add(*i)
if options.server_replay:
self.start_server_playback(
self._readflow(options.server_replay),
options.kill, options.rheaders,
not options.keepserving,
options.nopop,
options.replay_ignore_params,
options.replay_ignore_content,
options.replay_ignore_payload_params,
options.replay_ignore_host
)
if options.client_replay:
self.start_client_playback(
self._readflow(options.client_replay),
not options.keepserving
)
scripts = options.scripts or []
for command in scripts:
try:
self.load_script(command, use_reloader=True)
except exceptions.ScriptException as e:
raise DumpError(str(e))
if options.rfile:
try:
self.load_flows_file(options.rfile)
except exceptions.FlowReadException as v:
self.add_event("Flow file corrupted.", "error")
raise DumpError(v)
if self.o.app:
self.start_app(self.o.app_host, self.o.app_port)
def start(context):
if len(sys.argv) != 2:
raise ValueError("Usage: -s 'filt.py FILTER'")
context.filter = filt.parse(sys.argv[1])
def test_wideops(self):
a = filt.parse("~hq 'header: qvalue'")
assert isinstance(a, filt.FHeadRequest)
self._dump(a)