def configure(self, updated):
if self.certstore and not any(x in updated
for x in ["confdir", "certs"]):
return
certstore_path = os.path.expanduser(ctx.options.confdir)
self.certstore = certs.CertStore.from_store(
path=certstore_path,
basename=CONF_BASENAME,
key_size=ctx.options.key_size,
passphrase=ctx.options.cert_passphrase.encode("utf8")
if ctx.options.cert_passphrase else None,
)
for certspec in ctx.options.certs:
parts = certspec.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
cert = os.path.expanduser(parts[1])
if not os.path.exists(cert):
raise exceptions.OptionsError(
f"Certificate file does not exist: {cert}")
try:
self.certstore.add_cert_file(
parts[0],
cert,
passphrase=ctx.options.cert_passphrase.encode("utf8")
if ctx.options.cert_passphrase else None,
)
except crypto.Error as e:
raise exceptions.OptionsError(
f"Invalid certificate format: {cert}") from e
def configure(self, updated):
opts = ctx.options
if opts.add_upstream_certs_to_client_chain and not opts.upstream_cert:
raise exceptions.OptionsError(
"add_upstream_certs_to_client_chain requires the upstream_cert option to be enabled."
)
if "mode" in updated:
mode = opts.mode
if mode.startswith("reverse:") or mode.startswith("upstream:"):
try:
server_spec.parse_with_mode(mode)
except ValueError as e:
raise exceptions.OptionsError(str(e)) from e
elif mode == "transparent":
if not platform.original_addr:
raise exceptions.OptionsError(
"Transparent mode not supported on this platform."
)
elif mode not in ["regular", "socks5"]:
raise exceptions.OptionsError(
"Invalid mode specification: %s" % mode
)
if "client_certs" in updated:
if opts.client_certs:
client_certs = os.path.expanduser(opts.client_certs)
if not os.path.exists(client_certs):
raise exceptions.OptionsError(
f"Client certificate path does not exist: {opts.client_certs}"
)
def configure(self, options, updated):
if "auth_nonanonymous" in updated:
self.nonanonymous = options.auth_nonanonymous
if "auth_singleuser" in updated:
if options.auth_singleuser:
parts = options.auth_singleuser.split(':')
if len(parts) != 2:
raise exceptions.OptionsError(
"Invalid single-user auth specification.")
self.singleuser = parts
else:
self.singleuser = None
if "auth_htpasswd" in updated:
if options.auth_htpasswd:
try:
self.htpasswd = passlib.apache.HtpasswdFile(
options.auth_htpasswd)
except (ValueError, OSError) as v:
raise exceptions.OptionsError(
"Could not open htpasswd file: %s" % v)
else:
self.htpasswd = None
if "mode" in updated:
self.mode = options.mode
if self.enabled():
if options.mode == "transparent":
raise exceptions.OptionsError(
"Proxy Authentication not supported in transparent mode.")
if options.mode == "socks5":
raise exceptions.OptionsError(
"Proxy Authentication not supported in SOCKS mode. "
"https://github.com/mitmproxy/mitmproxy/issues/738")
def configure(self, options, updated):
if "proxyauth" in updated:
self.nonanonymous = False
self.singleuser = None
self.htpasswd = None
if options.proxyauth:
if options.proxyauth == "any":
self.nonanonymous = True
elif options.proxyauth.startswith("@"):
p = options.proxyauth[1:]
try:
self.htpasswd = passlib.apache.HtpasswdFile(p)
except (ValueError, OSError) as v:
raise exceptions.OptionsError(
"Could not open htpasswd file: %s" % p)
else:
parts = options.proxyauth.split(':')
if len(parts) != 2:
raise exceptions.OptionsError(
"Invalid single-user auth specification.")
self.singleuser = parts
if "mode" in updated:
self.mode = options.mode
if self.enabled():
if options.mode == "transparent":
raise exceptions.OptionsError(
"Proxy Authentication not supported in transparent mode.")
if options.mode == "socks5":
raise exceptions.OptionsError(
"Proxy Authentication not supported in SOCKS mode. "
"https://github.com/mitmproxy/mitmproxy/issues/738")
def load_paths(opts, *paths):
"""
Load paths in order. Each path takes precedence over the previous
path. Paths that don't exist are ignored, errors raise an
OptionsError.
Returns a dictionary of unknown options.
"""
ret = {}
for p in paths:
p = os.path.expanduser(p)
if os.path.exists(p) and os.path.isfile(p):
with open(p, "rt", encoding="utf8") as f:
try:
txt = f.read()
except UnicodeDecodeError as e:
raise exceptions.OptionsError(
"Error reading %s: %s" % (p, e)
)
try:
ret.update(load(opts, txt))
except exceptions.OptionsError as e:
raise exceptions.OptionsError(
"Error reading %s: %s" % (p, e)
)
return ret
def configure(self, options, updated):
"""
.replacements is a list of tuples (fpat, rex, s):
fpatt: a string specifying a filter pattern.
rex: a regular expression, as bytes.
s: the replacement string, as bytes
"""
if self.optionName in updated:
lst = []
for rep in getattr(options, self.optionName):
if isinstance(rep, str):
fpatt, rex, s = parse_hook(rep)
else:
fpatt, rex, s = rep
flt = flowfilter.parse(fpatt)
if not flt:
raise exceptions.OptionsError(
"Invalid filter pattern: %s" % fpatt)
try:
re.compile(rex)
except re.error as e:
raise exceptions.OptionsError(
"Invalid regular expression: %s - %s" % (rex, str(e)))
lst.append((rex, s, flt))
self.lst = lst
def configure(self, updated):
"""
.replacements is a list of tuples (fpat, rex, s):
fpatt: a string specifying a filter pattern.
rex: a regular expression, as string.
s: the replacement string
"""
if "replacements" in updated:
lst = []
for rep in ctx.options.replacements:
fpatt, rex, s = parse_hook(rep)
flt = flowfilter.parse(fpatt)
if not flt:
raise exceptions.OptionsError(
"Invalid filter pattern: %s" % fpatt)
try:
# We should ideally escape here before trying to compile
re.compile(rex)
except re.error as e:
raise exceptions.OptionsError(
"Invalid regular expression: %s - %s" % (rex, str(e)))
if s.startswith("@") and not os.path.isfile(s[1:]):
raise exceptions.OptionsError(
"Invalid file path: {}".format(s[1:]))
lst.append((rex, s, flt))
self.lst = lst
def configure(self, updated):
if "confdir" not in updated and "certs" not in updated:
return
certstore_path = os.path.expanduser(ctx.options.confdir)
self.certstore = certs.CertStore.from_store(
path=certstore_path,
basename=CONF_BASENAME,
key_size=ctx.options.key_size,
passphrase=ctx.options.cert_passphrase.encode("utf8") if ctx.options.cert_passphrase else None,
)
if self.certstore.default_ca.has_expired():
ctx.log.warn(
"The mitmproxy certificate authority has expired!\n"
"Please delete all CA-related files in your ~/.mitmproxy folder.\n"
"The CA will be regenerated automatically after restarting mitmproxy.\n"
"See https://docs.mitmproxy.org/stable/concepts-certificates/ for additional help.",
)
for certspec in ctx.options.certs:
parts = certspec.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
cert = Path(parts[1]).expanduser()
if not cert.exists():
raise exceptions.OptionsError(f"Certificate file does not exist: {cert}")
try:
self.certstore.add_cert_file(
parts[0],
cert,
passphrase=ctx.options.cert_passphrase.encode("utf8") if ctx.options.cert_passphrase else None,
)
except ValueError as e:
raise exceptions.OptionsError(f"Invalid certificate format for {cert}: {e}") from e
def configure(self, updated):
opts = ctx.options
if opts.add_upstream_certs_to_client_chain and not opts.upstream_cert:
raise exceptions.OptionsError(
"The no-upstream-cert and add-upstream-certs-to-client-chain "
"options are mutually exclusive. If no-upstream-cert is enabled "
"then the upstream certificate is not retrieved before generating "
"the client certificate chain.")
if "body_size_limit" in updated and opts.body_size_limit:
try:
opts._processed["body_size_limit"] = human.parse_size(
opts.body_size_limit)
except ValueError as e:
raise exceptions.OptionsError(
"Invalid body size limit specification: %s" %
opts.body_size_limit)
if "mode" in updated:
mode = opts.mode
if mode.startswith("reverse:") or mode.startswith("upstream:"):
try:
server_spec.parse_with_mode(mode)
except ValueError as e:
raise exceptions.OptionsError(str(e)) from e
elif mode == "transparent":
if not platform.original_addr:
raise exceptions.OptionsError(
"Transparent mode not supported on this platform.")
elif mode not in ["regular", "socks5"]:
raise exceptions.OptionsError(
"Invalid mode specification: %s" % mode)
def configure(self, options: moptions.Options,
updated: typing.Any) -> None:
self.check_ignore = DynamicHostMatcher(options.ignore_hosts)
self.check_tcp = HostMatcher(options.tcp_hosts)
certstore_path = os.path.expanduser(options.confdir)
if not os.path.exists(os.path.dirname(certstore_path)):
raise exceptions.OptionsError(
"Certificate Authority parent directory does not exist: %s" %
os.path.dirname(certstore_path))
self.certstore = certs.CertStore.from_store(certstore_path,
CONF_BASENAME)
for c in options.certs:
parts = c.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
cert = os.path.expanduser(parts[1])
if not os.path.exists(cert):
raise exceptions.OptionsError(
"Certificate file does not exist: %s" % cert)
try:
self.certstore.add_cert_file(parts[0], cert)
except crypto.Error:
raise exceptions.OptionsError(
"Invalid certificate format: %s" % cert)
m = options.mode
if m.startswith("upstream:") or m.startswith("reverse:"):
_, spec = server_spec.parse_with_mode(options.mode)
self.upstream_server = spec
def parse_setval(self, o: _Option, optstr: typing.Optional[str]) -> typing.Any:
"""
Convert a string to a value appropriate for the option type.
"""
if o.typespec in (str, typing.Optional[str]):
return optstr
elif o.typespec in (int, typing.Optional[int]):
if optstr:
try:
return int(optstr)
except ValueError:
raise exceptions.OptionsError("Not an integer: %s" % optstr)
elif o.typespec == int:
raise exceptions.OptionsError("Option is required: %s" % o.name)
else:
return None
elif o.typespec == bool:
if optstr == "toggle":
return not o.current()
if not optstr or optstr == "true":
return True
elif optstr == "false":
return False
else:
raise exceptions.OptionsError(
"Boolean must be \"true\", \"false\", or have the value " "omitted (a synonym for \"true\")."
)
elif o.typespec == typing.Sequence[str]:
if not optstr:
return []
else:
return getattr(self, o.name) + [optstr]
raise NotImplementedError("Unsupported option type: %s", o.typespec)
def configure(self, options, updated):
if "scripts" in updated:
for s in options.scripts:
if options.scripts.count(s) > 1:
raise exceptions.OptionsError("Duplicate script: %s" % s)
for a in ctx.master.addons.chain[:]:
if isinstance(a, Script) and a.name not in options.scripts:
ctx.log.info("Un-loading script: %s" % a.name)
ctx.master.addons.remove(a)
# The machinations below are to ensure that:
# - Scripts remain in the same order
# - Scripts are listed directly after the script addon. This is
# needed to ensure that interactions with, for instance, flow
# serialization remains correct.
# - Scripts are not initialized un-necessarily. If only a
# script's order in the script list has changed, it should simply
# be moved.
current = {}
for a in ctx.master.addons.chain[:]:
if isinstance(a, Script):
current[a.name] = a
ctx.master.addons.chain.remove(a)
ordered = []
newscripts = []
for s in options.scripts:
if s in current:
ordered.append(current[s])
else:
ctx.log.info("Loading script: %s" % s)
try:
sc = Script(s)
except ValueError as e:
raise exceptions.OptionsError(str(e))
ordered.append(sc)
newscripts.append(sc)
ochain = ctx.master.addons.chain
pos = ochain.index(self)
ctx.master.addons.chain = ochain[:pos + 1] + ordered + ochain[pos +
1:]
for s in newscripts:
l = addonmanager.Loader(ctx.master)
ctx.master.addons.invoke_addon(s, "load", l)
if self.is_running:
# If we're already running, we configure and tell the addon
# we're up and running.
ctx.master.addons.invoke_addon(s, "configure", options,
options.keys())
ctx.master.addons.invoke_addon(s, "running")
def configure(self, options: moptions.Options, updated: Any) -> None:
if options.add_upstream_certs_to_client_chain and not options.ssl_insecure:
raise exceptions.OptionsError(
"The verify-upstream-cert requires certificate verification to be disabled. "
"If upstream certificates are verified then extra upstream certificates are "
"not available for inclusion to the client chain.")
if options.ssl_insecure:
self.openssl_verification_mode_server = SSL.VERIFY_NONE
else:
self.openssl_verification_mode_server = SSL.VERIFY_PEER
self.check_ignore = HostMatcher(options.ignore_hosts)
self.check_tcp = HostMatcher(options.tcp_hosts)
self.openssl_method_client, self.openssl_options_client = \
tcp.sslversion_choices[options.ssl_version_client]
self.openssl_method_server, self.openssl_options_server = \
tcp.sslversion_choices[options.ssl_version_server]
certstore_path = os.path.expanduser(options.cadir)
if not os.path.exists(os.path.dirname(certstore_path)):
raise exceptions.OptionsError(
"Certificate Authority parent directory does not exist: %s" %
os.path.dirname(options.cadir))
self.certstore = certs.CertStore.from_store(certstore_path,
CONF_BASENAME)
if options.client_certs:
client_certs = os.path.expanduser(options.client_certs)
if not os.path.exists(client_certs):
raise exceptions.OptionsError(
"Client certificate path does not exist: %s" %
options.client_certs)
self.client_certs = client_certs
for c in options.certs:
parts = c.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
cert = os.path.expanduser(parts[1])
if not os.path.exists(cert):
raise exceptions.OptionsError(
"Certificate file does not exist: %s" % cert)
try:
self.certstore.add_cert_file(parts[0], cert)
except crypto.Error:
raise exceptions.OptionsError(
"Invalid certificate format: %s" % cert)
m = options.mode
if m.startswith("upstream:") or m.startswith("reverse:"):
_, spec = server_spec.parse_with_mode(options.mode)
self.upstream_server = spec
def configure(self, updated):
if "client_replay" in updated and ctx.options.client_replay:
try:
flows = io.read_flows_from_paths(ctx.options.client_replay)
except exceptions.FlowReadException as e:
raise exceptions.OptionsError(str(e))
self.start_replay(flows)
if "client_replay_concurrency" in updated:
if ctx.options.client_replay_concurrency not in [-1, 1]:
raise exceptions.OptionsError("Currently the only valid client_replay_concurrency values are -1 and 1.")
def _load(self, text):
if not text:
return {}
try:
data = ruamel.yaml.load(text, ruamel.yaml.RoundTripLoader)
except ruamel.yaml.error.YAMLError as v:
snip = v.problem_mark.get_snippet()
raise exceptions.OptionsError(
"Config error at line %s:\n%s\n%s" %
(v.problem_mark.line + 1, snip, v.problem))
if isinstance(data, str):
raise exceptions.OptionsError("Config error - no keys found.")
return data
def parse_hook(s):
"""
Returns a (pattern, regex, replacement) tuple.
The general form for a replacement hook is as follows:
/patt/regex/replacement
The first character specifies the separator. Example:
:~q:foo:bar
If only two clauses are specified, the pattern is set to match
universally (i.e. ".*"). Example:
/foo/bar/
Clauses are parsed from left to right. Extra separators are taken to be
part of the final clause. For instance, the replacement clause below is
"foo/bar/":
/one/two/foo/bar/
"""
sep, rem = s[0], s[1:]
parts = rem.split(sep, 2)
if len(parts) == 2:
patt = ".*"
a, b = parts
elif len(parts) == 3:
patt, a, b = parts
else:
raise exceptions.OptionsError("Invalid replacement specifier: %s" % s)
return patt, a, b
def parse_spec(
spec: str) -> Tuple[bool, str, Optional[int], str, str, str]:
try:
if spec.count(":") > 4:
(
security,
url,
port_str,
ldap_user,
ldap_pass,
dn_subtree,
) = spec.split(":")
port = int(port_str)
else:
security, url, ldap_user, ldap_pass, dn_subtree = spec.split(
":")
port = None
if security == "ldaps":
use_ssl = True
elif security == "ldap":
use_ssl = False
else:
raise ValueError
return use_ssl, url, port, ldap_user, ldap_pass, dn_subtree
except ValueError:
raise exceptions.OptionsError(
f"Invalid LDAP specification: {spec}")
def configure(self, updated):
if "stream_large_bodies" in updated and ctx.options.stream_large_bodies:
try:
self.max_size = human.parse_size(
ctx.options.stream_large_bodies)
except ValueError as e:
raise exceptions.OptionsError(e)
def configure(self, updated):
if "client_replay" in updated and ctx.options.client_replay:
try:
flows = io.read_flows_from_paths(ctx.options.client_replay)
except exceptions.FlowReadException as e:
raise exceptions.OptionsError(str(e))
self.start_replay(flows)
def __init__(self, proxyauth: str):
path = proxyauth[1:]
try:
self.htpasswd = passlib.apache.HtpasswdFile(path)
except (ValueError, OSError):
raise exceptions.OptionsError(
f"Could not open htpasswd file: {path}")
def parse(text):
if not text:
return {}
try:
data = ruamel.yaml.load(text, ruamel.yaml.RoundTripLoader)
except ruamel.yaml.error.YAMLError as v:
if hasattr(v, "problem_mark"):
snip = v.problem_mark.get_snippet()
raise exceptions.OptionsError(
"Config error at line %s:\n%s\n%s" %
(v.problem_mark.line + 1, snip, v.problem))
else:
raise exceptions.OptionsError("Could not parse options.")
if isinstance(data, str):
raise exceptions.OptionsError("Config error - no keys found.")
return data
def subscribe(self, func, opts):
"""
Subscribe a callable to the .changed signal, but only for a
specified list of options. The callable should accept arguments
(options, updated), and may raise an OptionsError.
The event will automatically be unsubscribed if the callable goes out of scope.
"""
for i in opts:
if i not in self._options:
raise exceptions.OptionsError("No such option: %s" % i)
# We reuse blinker's safe reference functionality to cope with weakrefs
# to bound methods.
func = blinker._saferef.safe_ref(func)
@functools.wraps(func)
def _call(options, updated):
if updated.intersection(set(opts)):
f = func()
if f:
f(options, updated)
else:
self.changed.disconnect(_call)
# Our wrapper function goes out of scope immediately, so we have to set
# weakrefs to false. This means we need to keep our own weakref, and
# clean up the hook when it's gone.
self.changed.connect(_call, weak=False)
def __init__(self, proxyauth: str):
try:
security, url, ldap_user, ldap_pass, self.dn_subtree = proxyauth.split(
":")
except ValueError:
raise exceptions.OptionsError("Invalid ldap specification")
if security == "ldaps":
server = ldap3.Server(url, use_ssl=True)
elif security == "ldap":
server = ldap3.Server(url)
else:
raise exceptions.OptionsError(
"Invalid ldap specification on the first part")
conn = ldap3.Connection(server, ldap_user, ldap_pass, auto_bind=True)
self.conn = conn
self.server = server
def parse_upstream_auth(auth):
pattern = re.compile(".+:")
if pattern.search(auth) is None:
raise exceptions.OptionsError(
"Invalid upstream auth specification: %s" % auth
)
return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
def start_stream_to_path(self, path, flt):
try:
f = self.open_file(path)
except IOError as v:
raise exceptions.OptionsError(str(v))
self.stream = io.FilteredFlowWriter(f, flt)
self.active_flows = set()
def __init__(
self,
options: options.Options,
server,
with_termlog=True,
with_dumper=True,
) -> None:
master.Master.__init__(self, options, server)
self.has_errored = False
if with_termlog:
self.addons.add(termlog.TermLog())
self.addons.add(*addons.default_addons())
if with_dumper:
self.addons.add(dumper.Dumper())
if self.options.server:
self.add_log(
"Proxy server listening at http://{}:{}".format(
server.address[0], server.address[1]), "info")
if options.rfile:
try:
self.load_flows_file(options.rfile)
except exceptions.FlowReadException as v:
self.add_log("Flow file corrupted.", "error")
raise exceptions.OptionsError(v)
def set(self, *spec, defer=False):
"""
Takes a list of set specification in standard form (option=value).
Options that are known are updated immediately. If defer is true,
options that are not known are deferred, and will be set once they
are added.
"""
vals = {}
unknown = {}
for i in spec:
parts = i.split("=", maxsplit=1)
if len(parts) == 1:
optname, optval = parts[0], None
else:
optname, optval = parts[0], parts[1]
if optname in self._options:
vals[optname] = self.parse_setval(self._options[optname],
optval)
else:
unknown[optname] = optval
if defer:
self.deferred.update(unknown)
elif unknown:
raise exceptions.OptionsError("Unknown options: %s" %
", ".join(unknown.keys()))
self.update(**vals)
async def doread(self, rfile):
try:
await self.load_flows_from_path(ctx.options.rfile)
except exceptions.FlowReadException as e:
raise exceptions.OptionsError(e) from e
finally:
ctx.master.addons.trigger("processing_complete")
def parse_modify_headers(s):
"""
Returns a (header_name, header_value, flow_filter) tuple.
The general form for a modify_headers hook is as follows:
/header_name/header_value/flow_filter
The first character specifies the separator. Example:
:foo:bar:~q
If only two clauses are specified, the pattern is set to match
universally (i.e. ".*"). Example:
/foo/bar/
Clauses are parsed from left to right. Extra separators are taken to be
part of the final clause. For instance, the flow filter below is
"foo/bar/":
/one/two/foo/bar/
"""
sep, rem = s[0], s[1:]
parts = rem.split(sep, 2)
if len(parts) == 2:
flow_filter = ".*"
header_name, header_value = parts
elif len(parts) == 3:
header_name, header_value, flow_filter = parts
else:
raise exceptions.OptionsError("Invalid replacement specifier: %s" % s)
return header_name, header_value, flow_filter
def configure(self, options: moptions.Options, updated: typing.Any) -> None:
if options.allow_hosts and options.ignore_hosts:
raise exceptions.OptionsError("--ignore-hosts and --allow-hosts are mutually "
"exclusive; please choose one.")
if options.ignore_hosts:
self.check_filter = HostMatcher("ignore", options.ignore_hosts)
elif options.allow_hosts:
self.check_filter = HostMatcher("allow", options.allow_hosts)
else:
self.check_filter = HostMatcher(False)
if "tcp_hosts" in updated:
self.check_tcp = HostMatcher("tcp", options.tcp_hosts)
certstore_path = os.path.expanduser(options.confdir)
if not os.path.exists(os.path.dirname(certstore_path)):
raise exceptions.OptionsError(
"Certificate Authority parent directory does not exist: %s" %
os.path.dirname(certstore_path)
)
key_size = options.key_size
self.certstore = certs.CertStore.from_store(
certstore_path,
moptions.CONF_BASENAME,
key_size
)
for c in options.certs:
parts = c.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
cert = os.path.expanduser(parts[1])
if not os.path.exists(cert):
raise exceptions.OptionsError(
"Certificate file does not exist: %s" % cert
)
try:
self.certstore.add_cert_file(parts[0], cert)
except crypto.Error:
raise exceptions.OptionsError(
"Invalid certificate format: %s" % cert
)
m = options.mode
if m.startswith("upstream:") or m.startswith("reverse:"):
_, spec = server_spec.parse_with_mode(options.mode)
self.upstream_server = spec
