def start_response(status, headers, exc_info=None): if exc_info: if state["headers_sent"]: raise exc_info[1] elif state["status"]: raise AssertionError('Response already started') state["status"] = status state["headers"] = http.Headers([[strutils.always_bytes(k), strutils.always_bytes(v)] for k, v in headers]) if exc_info: self.error_page(soc, state["headers_sent"], traceback.format_tb(exc_info[2])) state["headers_sent"] = True
def make( cls, method: str, url: str, content: Union[bytes, str] = "", headers: Union[Headers, Dict[Union[str, bytes], Union[str, bytes]], Iterable[Tuple[bytes, bytes]]] = () ) -> "Request": """ Simplified API for creating request objects. """ # Headers can be list or dict, we differentiate here. if isinstance(headers, Headers): pass elif isinstance(headers, dict): headers = Headers((always_bytes(k, "utf-8", "surrogateescape"), always_bytes(v, "utf-8", "surrogateescape")) for k, v in headers.items()) elif isinstance(headers, Iterable): headers = Headers(headers) else: raise TypeError( "Expected headers to be an iterable or dict, but is {}.". format(type(headers).__name__)) req = cls( "", 0, method.encode("utf-8", "surrogateescape"), b"", b"", b"", b"HTTP/1.1", headers, b"", None, time.time(), time.time(), ) req.url = url # Assign this manually to update the content-length header. if isinstance(content, bytes): req.content = content elif isinstance(content, str): req.text = content else: raise TypeError( f"Expected content to be str or bytes, but is {type(content).__name__}." ) return req
def parse_webform(flow): r = flow.request _, netloc, path, _, query, _ = urllib.parse.urlparse(r.url) # these parameters will be ignored during matching. to_bytes = lambda params: [strutils.always_bytes(i) for i in params] queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) headers = dict(r.data.headers) form_items = [] # matches json-type responses. for example, # application/json;charset=UTF-8 # application/xxx-json. content_type = (headers.get('content-type', '') or headers.get('Content-Type', '')) if re.match(r'application/.*json.*', content_type): form_unsored = json.loads(r.data.content.decode('utf-8')) form_items = [(k, json.dumps(form_unsored[k], sort_keys=True)) for k in sorted(form_unsored.keys())] else: form_contents = r.urlencoded_form or r.multipart_form log_warn('form contents = %s', form_contents) if form_contents: form_items = form_contents.items(multi=True) form = dict(form_items) return form
def parse_upstream_auth(auth): pattern = re.compile(".+:") if pattern.search(auth) is None: raise exceptions.OptionsError( "Invalid upstream auth specification: %s" % auth ) return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
def make( cls, status_code: int = 200, content: Union[bytes, str] = b"", headers: Union[Headers, Mapping[str, Union[str, bytes]], Iterable[Tuple[bytes, bytes]]] = () ) -> "Response": """ Simplified API for creating response objects. """ if isinstance(headers, Headers): headers = headers elif isinstance(headers, dict): headers = Headers(( always_bytes(k, "utf-8", "surrogateescape"), # type: ignore always_bytes(v, "utf-8", "surrogateescape")) for k, v in headers.items()) elif isinstance(headers, Iterable): headers = Headers(headers) # type: ignore else: raise TypeError( "Expected headers to be an iterable or dict, but is {}.". format(type(headers).__name__)) resp = cls( b"HTTP/1.1", status_code, status_codes.RESPONSES.get(status_code, "").encode(), headers, None, None, time.time(), time.time(), ) # Assign this manually to update the content-length header. if isinstance(content, bytes): resp.content = content elif isinstance(content, str): resp.text = content else: raise TypeError( f"Expected content to be str or bytes, but is {type(content).__name__}." ) return resp
def test_always_bytes(): assert strutils.always_bytes(bytes(range(256))) == bytes(range(256)) assert strutils.always_bytes("foo") == b"foo" with tutils.raises(ValueError): strutils.always_bytes(u"\u2605", "ascii") with tutils.raises(TypeError): strutils.always_bytes(42, "ascii")
def test_always_bytes(): assert strutils.always_bytes(bytes(range(256))) == bytes(range(256)) assert strutils.always_bytes("foo") == b"foo" with pytest.raises(ValueError): strutils.always_bytes(u"\u2605", "ascii") with pytest.raises(TypeError): strutils.always_bytes(42, "ascii")
def values(self, settings): if self.body: bodygen = self.body.value.get_generator(settings) length = len(self.body.value.get_generator(settings)) elif self.rawbody: bodygen = self.rawbody.value.get_generator(settings) length = len(self.rawbody.value.get_generator(settings)) elif self.nested_frame: bodygen = NESTED_LEADER + strutils.always_bytes(self.nested_frame.parsed.spec()) length = len(bodygen) else: bodygen = None length = 0 if self.toklength: length = int(self.toklength.value) frameparts = dict( payload_length=length ) if self.mask and self.mask.value: frameparts["mask"] = True if self.knone: frameparts["masking_key"] = None elif self.key: key = self.key.values(settings)[0][:] frameparts["masking_key"] = key for i in ["opcode", "fin", "rsv1", "rsv2", "rsv3", "mask"]: v = getattr(self, i, None) if v is not None: frameparts[i] = v.value # import wsproto.frame_protocol # wsproto.frame_protocol.Frame( # opcode=frameparts["opcode"], # payload=None, # frame_finished=frameparts["fin"] # ) frame = websockets_frame.FrameHeader(**frameparts) vals = [bytes(frame)] if bodygen: if frame.masking_key and not self.rawbody: masker = websockets_frame.Masker(frame.masking_key) vals.append( generators.TransformGenerator( bodygen, masker.mask ) ) else: vals.append(bodygen) return vals
def _hash(self, flow): """ Calculates a loose hash of the flow request. """ r = flow.request _, _, path, _, query, _ = urllib.parse.urlparse(r.url) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) key = [str(r.port), str(r.scheme), str(r.method), str(path)] if not self.options.server_replay_ignore_content: form_contents = r.urlencoded_form or r.multipart_form if self.options.server_replay_ignore_payload_params and form_contents: params = [ strutils.always_bytes(i) for i in self.options.server_replay_ignore_payload_params ] for p in form_contents.items(multi=True): if p[0] not in params: key.append(p) else: key.append(str(r.raw_content)) if not self.options.server_replay_ignore_host: key.append(r.host) filtered = [] ignore_params = self.options.server_replay_ignore_params or [] for p in queriesArray: if p[0] not in ignore_params: filtered.append(p) for p in filtered: key.append(p[0]) key.append(p[1]) if self.options.server_replay_use_headers: headers = [] for i in self.options.server_replay_use_headers: v = r.headers.get(i) headers.append((i, v)) key.append(headers) return hashlib.sha256( repr(key).encode("utf8", "surrogateescape") ).digest()
def values(self, settings): if self.body: bodygen = self.body.value.get_generator(settings) length = len(self.body.value.get_generator(settings)) elif self.rawbody: bodygen = self.rawbody.value.get_generator(settings) length = len(self.rawbody.value.get_generator(settings)) elif self.nested_frame: bodygen = NESTED_LEADER + strutils.always_bytes(self.nested_frame.parsed.spec()) length = len(bodygen) else: bodygen = None length = 0 if self.toklength: length = int(self.toklength.value) frameparts = dict( payload_length=length ) if self.mask and self.mask.value: frameparts["mask"] = True if self.knone: frameparts["masking_key"] = None elif self.key: key = self.key.values(settings)[0][:] frameparts["masking_key"] = key for i in ["opcode", "fin", "rsv1", "rsv2", "rsv3", "mask"]: v = getattr(self, i, None) if v is not None: frameparts[i] = v.value frame = mitmproxy.net.websockets.FrameHeader(**frameparts) vals = [bytes(frame)] if bodygen: if frame.masking_key and not self.rawbody: masker = mitmproxy.net.websockets.Masker(frame.masking_key) vals.append( generators.TransformGenerator( bodygen, masker.mask ) ) else: vals.append(bodygen) return vals
def _hash(self, flow): """ Calculates a loose hash of the flow request. """ r = flow.request _, _, path, _, query, _ = urllib.parse.urlparse(r.url) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) key = [str(r.port), str(r.scheme), str(r.method), str(path)] if not self.options.server_replay_ignore_content: form_contents = r.urlencoded_form or r.multipart_form if self.options.server_replay_ignore_payload_params and form_contents: params = [ strutils.always_bytes(i) for i in self.options.server_replay_ignore_payload_params ] for p in form_contents.items(multi=True): if p[0] not in params: key.append(p) else: key.append(str(r.raw_content)) if not self.options.server_replay_ignore_host: key.append(r.host) filtered = [] ignore_params = self.options.server_replay_ignore_params or [] for p in queriesArray: if p[0] not in ignore_params: filtered.append(p) for p in filtered: key.append(p[0]) key.append(p[1]) if self.options.server_replay_use_headers: headers = [] for i in self.options.server_replay_use_headers: v = r.headers.get(i) headers.append((i, v)) key.append(headers) return hashlib.sha256(repr(key).encode("utf8", "surrogateescape")).digest()
def method(self, method): self.data.method = strutils.always_bytes(method, "utf-8", "surrogateescape")
def scheme(self, val: Union[str, bytes]) -> None: self.data.scheme = always_bytes(val, "utf-8", "surrogateescape")
def path(self, val: Union[str, bytes]) -> None: self.data.path = always_bytes(val, "utf-8", "surrogateescape")
def _hash(self, flow): ''' Calculates a loose hash of the flow request. ''' r = flow.request _, netloc, path, _, query, _ = urllib.parse.urlparse(r.url) # these parameters will be ignored during matching. to_bytes = lambda params: [strutils.always_bytes(i) for i in params] ignore_site_params = IGNORE_SITE_PARAMS.get(netloc, []) ignore_url_params = IGNORE_URL_PARAMS.get(netloc + path, []) whitelist_url_params = WHITELIST_URL_PARAMS.get(netloc + path, None) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) key = [str(r.port), str(r.scheme), str(r.method), str(path)] headers = dict(r.data.headers) form_contents = r.urlencoded_form or r.multipart_form form_items = [] if re.match(r'application/.*json', headers.get('content-type', '')): form_unsorted = json.loads(r.data.content.decode('utf-8')) if isinstance(form_unsorted, list): # make sure form is a dict. form_unsorted = {'data': form_unsorted} form_items = [(k, json.dumps(form_unsorted[k], sort_keys=True)) for k in sorted(form_unsorted.keys())] elif form_contents: form_items = form_contents.items(multi=True) if form_items: for p in form_items: param = strutils.always_bytes(p[0]) if param in to_bytes(ignore_site_params) or param in to_bytes( ignore_url_params): logger.warn('[proxy hash] ignoring parameter %s', str(p[0])) continue if whitelist_url_params and param not in to_bytes( whitelist_url_params): logger.warn('[proxy hash] ignoring parameter %s', str(p[0])) continue key.append(p) else: key.append(str(r.raw_content)) if not self.options.server_replay_ignore_host: key.append(r.host) filtered = [] for p in queriesArray: param = p[0] if param in ignore_site_params or param in ignore_url_params: logger.warn('[proxy hash] ignoring parameter %s', str(p[0])) continue if whitelist_url_params and param not in whitelist_url_params: logger.warn('[proxy hash] ignoring parameter %s', str(p[0])) continue filtered.append(p) for p in filtered: key.append(p[0]) key.append(p[1]) if self.options.server_replay_use_headers: headers = [] for i in self.options.server_replay_use_headers: v = r.headers.get(i) headers.append((i, v)) key.append(headers) binary = hashlib.sha256(repr(key).encode("utf8", "surrogateescape")).digest() if r.method == 'POST': logger.warn('\033[91m Args %s \033[0m', repr(key).encode("utf8", "surrogateescape")) return str(base64.b64encode(binary), 'utf8')
def method(self, val: Union[str, bytes]) -> None: self.data.method = always_bytes(val, "utf-8", "surrogateescape")
def http_version(self, http_version): self.data.http_version = strutils.always_bytes(http_version, "utf-8", "surrogateescape")
def create_server_nonce(client_nonce): return base64.b64encode( hashlib.sha1(strutils.always_bytes(client_nonce) + MAGIC).digest())
def reason(self, reason: Union[str, bytes]) -> None: self.data.reason = strutils.always_bytes(reason, "ISO-8859-1")
def reason(self, reason): self.data.reason = strutils.always_bytes(reason, "ISO-8859-1", "surrogateescape")
def path(self, path): self.data.path = strutils.always_bytes(path, "utf-8", "surrogateescape")
def create_server_nonce(client_nonce): return base64.b64encode(hashlib.sha1(strutils.always_bytes(client_nonce) + MAGIC).digest())
def scheme(self, scheme): self.data.scheme = strutils.always_bytes(scheme, "utf-8", "surrogateescape")
def http_version(self, http_version: Union[str, bytes]) -> None: self.data.http_version = strutils.always_bytes(http_version, "utf-8", "surrogateescape")
def _always_bytes(x): return strutils.always_bytes(x, "utf-8", "surrogateescape")
def _always_bytes(x: Union[str, bytes]) -> bytes: return strutils.always_bytes(x, "utf-8", "surrogateescape")
def flowdetails(state, flow: mitmproxy.flow.Flow): text = [] sc = flow.server_conn cc = flow.client_conn req: typing.Optional[http.HTTPRequest] resp: typing.Optional[http.HTTPResponse] if isinstance(flow, http.HTTPFlow): req = flow.request resp = flow.response else: req = None resp = None metadata = flow.metadata if metadata is not None and len(metadata) > 0: parts = [(str(k), repr(v)) for k, v in metadata.items()] text.append(urwid.Text([("head", "Metadata:")])) text.extend(common.format_keyvals(parts, indent=4)) if sc is not None and sc.ip_address: text.append(urwid.Text([("head", "Server Connection:")])) parts = [ ("Address", human.format_address(sc.address)), ] if sc.ip_address: parts.append( ("Resolved Address", human.format_address(sc.ip_address))) if resp: parts.append(("HTTP Version", resp.http_version)) if sc.alpn_proto_negotiated: parts.append( ("ALPN", strutils.bytes_to_escaped_str(sc.alpn_proto_negotiated))) text.extend(common.format_keyvals(parts, indent=4)) c = sc.cert if c: text.append(urwid.Text([("head", "Server Certificate:")])) parts = [("Type", "%s, %s bits" % c.keyinfo), ("SHA1 digest", c.digest("sha1")), ("Valid to", str(c.notafter)), ("Valid from", str(c.notbefore)), ("Serial", str(c.serial)), ("Subject", urwid.BoxAdapter( urwid.ListBox( common.format_keyvals(c.subject, key_format="highlight")), len(c.subject))), ("Issuer", urwid.BoxAdapter( urwid.ListBox( common.format_keyvals(c.issuer, key_format="highlight")), len(c.issuer)))] if c.altnames: parts.append(("Alt names", ", ".join( strutils.bytes_to_escaped_str(x) for x in c.altnames))) text.extend(common.format_keyvals(parts, indent=4)) if cc is not None: text.append(urwid.Text([("head", "Client Connection:")])) parts = [ ("Address", "{}:{}".format(cc.address[0], cc.address[1])), ] if req: parts.append(("HTTP Version", req.http_version)) if cc.tls_version: parts.append(("TLS Version", cc.tls_version)) if cc.sni: parts.append(("Server Name Indication", strutils.bytes_to_escaped_str( strutils.always_bytes(cc.sni, "idna")))) if cc.cipher_name: parts.append(("Cipher Name", cc.cipher_name)) if cc.alpn_proto_negotiated: parts.append( ("ALPN", strutils.bytes_to_escaped_str(cc.alpn_proto_negotiated))) text.extend(common.format_keyvals(parts, indent=4)) parts = [] if cc is not None and cc.timestamp_start: parts.append(("Client conn. established", maybe_timestamp(cc, "timestamp_start"))) if cc.tls_established: parts.append(("Client conn. TLS handshake", maybe_timestamp(cc, "timestamp_tls_setup"))) parts.append( ("Client conn. closed", maybe_timestamp(cc, "timestamp_end"))) if sc is not None and sc.timestamp_start: parts.append( ("Server conn. initiated", maybe_timestamp(sc, "timestamp_start"))) parts.append(("Server conn. TCP handshake", maybe_timestamp(sc, "timestamp_tcp_setup"))) if sc.tls_established: parts.append(("Server conn. TLS handshake", maybe_timestamp(sc, "timestamp_tls_setup"))) parts.append( ("Server conn. closed", maybe_timestamp(sc, "timestamp_end"))) if req is not None and req.timestamp_start: parts.append( ("First request byte", maybe_timestamp(req, "timestamp_start"))) parts.append( ("Request complete", maybe_timestamp(req, "timestamp_end"))) if resp is not None and resp.timestamp_start: parts.append( ("First response byte", maybe_timestamp(resp, "timestamp_start"))) parts.append( ("Response complete", maybe_timestamp(resp, "timestamp_end"))) if parts: # sort operations by timestamp parts = sorted(parts, key=lambda p: p[1]) text.append(urwid.Text([("head", "Timing:")])) text.extend(common.format_keyvals(parts, indent=4)) return searchable.Searchable(text)