def clip(
self,
flows: typing.Sequence[flow.Flow],
cuts: mitmproxy.types.CutSpec,
) -> None:
"""
Send cuts to the clipboard. If there are multiple flows or cuts, the
format is UTF-8 encoded CSV. If there is exactly one row and one
column, the data is written to file as-is, with raw bytes preserved.
"""
fp = io.StringIO(newline="")
if len(cuts) == 1 and len(flows) == 1:
v = extract(cuts[0], flows[0])
if isinstance(v, bytes):
fp.write(strutils.always_str(v))
else:
fp.write("utf8")
ctx.log.alert("Clipped single cut.")
else:
writer = csv.writer(fp)
for f in flows:
vals = [extract(c, f) for c in cuts]
writer.writerow(
[strutils.always_str(v) or "" for v in vals] # type: ignore
)
ctx.log.alert("Clipped %s cuts as CSV." % len(cuts))
try:
pyperclip.copy(fp.getvalue())
except pyperclip.PyperclipException as e:
ctx.log.error(str(e))
def clip(
self,
flows: typing.Sequence[flow.Flow],
cuts: mitmproxy.types.CutSpec,
) -> None:
"""
Send cuts to the clipboard. If there are multiple flows or cuts, the
format is UTF-8 encoded CSV. If there is exactly one row and one
column, the data is written to file as-is, with raw bytes preserved.
"""
fp = io.StringIO(newline="")
if len(cuts) == 1 and len(flows) == 1:
v = extract(cuts[0], flows[0])
if isinstance(v, bytes):
fp.write(strutils.always_str(v))
else:
fp.write(v)
ctx.log.alert("Clipped single cut.")
else:
writer = csv.writer(fp)
for f in flows:
vals = [extract(c, f) for c in cuts]
writer.writerow(
[strutils.always_str(v) or "" for v in vals] # type: ignore
)
ctx.log.alert("Clipped %s cuts as CSV." % len(cuts))
try:
pyperclip.copy(fp.getvalue())
except pyperclip.PyperclipException as e:
ctx.log.error(str(e))
def code_clip(self, flows: typing.Sequence[flow.Flow]) -> None:
"""Export a flow to the system clipboard as locust code."""
ctx.log.info(str(type(flows)))
data = strutils.always_str(self.context.locust_code(flows[0]))
if len(flows) > 1:
for f in flows[1:]:
v = strutils.always_str(self.context.locust_task(f))
tmp = data[:-100]
tmp += v + '\n'
tmp += data[-100:]
data = tmp
try:
pyperclip.copy(data)
except pyperclip.PyperclipException as e:
ctx.log.error(str(e))
def parse_authority(authority: AnyStr, check: bool) -> Tuple[str, Optional[int]]:
"""Extract the host and port from host header/authority information
Raises:
ValueError, if check is True and the authority information is malformed.
"""
try:
if isinstance(authority, bytes):
authority_str = authority.decode("idna")
else:
authority_str = authority
m = _authority_re.match(authority_str)
if not m:
raise ValueError
host = m.group("host")
if host.startswith("[") and host.endswith("]"):
host = host[1:-1]
if not is_valid_host(host):
raise ValueError
if m.group("port"):
port = int(m.group("port"))
if not is_valid_port(port):
raise ValueError
return host, port
else:
return host, None
except ValueError:
if check:
raise
else:
return always_str(authority, "utf-8", "surrogateescape"), None
def save(self, cuts: command.Cuts, path: command.Path) -> None:
"""
Save cuts to file. If there are multiple rows or columns, the format
is UTF-8 encoded CSV. If there is exactly one row and one column,
the data is written to file as-is, with raw bytes preserved. If the
path is prefixed with a "+", values are appended if there is an
existing file.
"""
append = False
if path.startswith("+"):
append = True
path = command.Path(path[1:])
if len(cuts) == 1 and len(cuts[0]) == 1:
with open(path, "ab" if append else "wb") as fp:
if fp.tell() > 0:
# We're appending to a file that already exists and has content
fp.write(b"\n")
v = cuts[0][0]
if isinstance(v, bytes):
fp.write(v)
else:
fp.write(v.encode("utf8"))
ctx.log.alert("Saved single cut.")
else:
with open(path,
"a" if append else "w",
newline='',
encoding="utf8") as fp:
writer = csv.writer(fp)
for r in cuts:
writer.writerow([strutils.always_str(c) or ""
for c in r] # type: ignore
)
ctx.log.alert("Saved %s cuts as CSV." % len(cuts))
def clip(self, format: str, flow: flow.Flow) -> None:
"""
Export a flow to the system clipboard.
"""
if format not in formats:
raise exceptions.CommandError("No such export format: %s" % format)
func: typing.Any = formats[format]
if format == "curl":
v = strutils.always_str(
func(flow,
preserve_ip=ctx.options.export_preserve_original_ip))
else:
v = strutils.always_str(func(flow))
try:
pyperclip.copy(v)
except pyperclip.PyperclipException as e:
ctx.log.error(str(e))
def _convert_dict_keys(o: Any) -> Any:
if isinstance(o, dict):
return {
strutils.always_str(k): _convert_dict_keys(v)
for k, v in o.items()
}
else:
return o
def _convert_dict_vals(o: dict, values_to_convert: dict) -> dict:
for k, v in values_to_convert.items():
if not o or k not in o:
continue # pragma: no cover
if v is True:
o[k] = strutils.always_str(o[k])
else:
_convert_dict_vals(o[k], v)
return o
def clip(self, fmt: str, f: flow.Flow) -> None:
"""
Export a flow to the system clipboard.
"""
if fmt not in formats:
raise exceptions.CommandError("No such export format: %s" % fmt)
func = formats[fmt] # type: typing.Any
v = strutils.always_str(func(f))
pyperclip.copy(v)
def export_str(self, format: str, f: flow.Flow) -> str:
"""
Export a flow and return the result.
"""
if format not in formats:
raise exceptions.CommandError("No such export format: %s" % format)
func = formats[format]
return strutils.always_str(func(f), "utf8", "backslashreplace")
def host(self, val: Union[str, bytes]) -> None:
self.data.host = always_str(val, "idna", "strict")
# Update host header
if "Host" in self.data.headers:
self.data.headers["Host"] = val
# Update authority
if self.data.authority:
self.authority = url.hostport(self.scheme, self.host, self.port)
def _convert_dict_vals(o: dict, values_to_convert: dict) -> dict:
for k, v in values_to_convert.items():
if not o or k not in o:
continue # pragma: no cover
if v is True:
o[k] = strutils.always_str(o[k])
else:
_convert_dict_vals(o[k], v)
return o
def dump(self, data, hexdump):
if hexdump:
for line in strutils.hexdump(data):
self("\t%s %s %s" % line)
else:
data = strutils.always_str(
strutils.escape_control_characters(
data.decode("ascii", "replace").replace(u"\ufffd", u".")))
for i in data.split("\n"):
self("\t%s" % i)
def verify_cert(conn, x509, errno, err_depth, is_cert_verified):
if not is_cert_verified:
self.ssl_verification_error = exceptions.InvalidCertificateException(
"Certificate Verification Error for {}: {} (errno: {}, depth: {})".format(
sni,
strutils.always_str(SSL._ffi.string(SSL._lib.X509_verify_cert_error_string(errno)), "utf8"),
errno,
err_depth
)
)
return is_cert_verified
def clip(self, cuts: command.Cuts) -> None:
"""
Send cuts to the system clipboard.
"""
fp = io.StringIO(newline="")
if len(cuts) == 1 and len(cuts[0]) == 1:
v = cuts[0][0]
if isinstance(v, bytes):
fp.write(strutils.always_str(v))
else:
fp.write("utf8")
ctx.log.alert("Clipped single cut.")
else:
writer = csv.writer(fp)
for r in cuts:
writer.writerow([strutils.always_str(c) or ""
for c in r] # type: ignore
)
ctx.log.alert("Clipped %s cuts as CSV." % len(cuts))
pyperclip.copy(fp.getvalue())
def verify_cert(conn, x509, errno, err_depth, is_cert_verified):
if not is_cert_verified:
self.ssl_verification_error = exceptions.InvalidCertificateException(
"Certificate Verification Error for {}: {} (errno: {}, depth: {})"
.format(
sni,
strutils.always_str(
SSL._ffi.string(
SSL._lib.X509_verify_cert_error_string(
errno)), "utf8"), errno, err_depth))
return is_cert_verified
def task_clip(self, flows: typing.Sequence[flow.Flow]) -> None:
"""Export a flow to the system clipboard as locust task."""
ctx.log.info(str(type(flows)))
data = ''
for f in flows:
v = strutils.always_str(self.context.locust_task(f))
data += v
try:
pyperclip.copy(data)
except pyperclip.PyperclipException as e:
ctx.log.error(str(e))
def grideditor_save(self, path: mitmproxy.types.Path) -> None:
"""
Save data to file as a CSV.
"""
rows = self._grideditor().value
with open(path, "w", newline='', encoding="utf8") as fp:
writer = csv.writer(fp)
for row in rows:
writer.writerow([strutils.always_str(x) or ""
for x in row] # type: ignore
)
ctx.log.alert("Saved %s rows as CSV." % (len(rows)))
def clip(self, fmt: str, f: flow.Flow) -> None:
"""
Export a flow to the system clipboard.
"""
if fmt not in formats:
raise exceptions.CommandError("No such export format: %s" % fmt)
func: typing.Any = formats[fmt]
v = strutils.always_str(func(f))
try:
pyperclip.copy(v)
except pyperclip.PyperclipException as e:
ctx.log.error(str(e))
def grideditor_save(self, path: mitmproxy.types.Path) -> None:
"""
Save data to file as a CSV.
"""
rows = self._grideditor().value
with open(path, "w", newline='', encoding="utf8") as fp:
writer = csv.writer(fp)
for row in rows:
writer.writerow(
[strutils.always_str(x) or "" for x in row] # type: ignore
)
ctx.log.alert("Saved %s rows as CSV." % (len(rows)))
def clip(self, format: str, f: flow.Flow) -> None:
"""
Export a flow to the system clipboard.
"""
if format not in formats:
raise exceptions.CommandError("No such export format: %s" % format)
func = formats[format]
val = strutils.always_str(func(f), "utf8", "backslashreplace")
try:
pyperclip.copy(val)
except pyperclip.PyperclipException as e:
ctx.log.error(str(e))
def dump(self, data, hexdump):
if hexdump:
for line in strutils.hexdump(data):
self("\t%s %s %s" % line)
else:
data = strutils.always_str(
strutils.escape_control_characters(
data
.decode("ascii", "replace")
.replace(u"\ufffd", u".")
)
)
for i in data.split("\n"):
self("\t%s" % i)
def __str__(self):
parts = [
"Application Layer Protocol: %s" % strutils.always_str(self.alp, "utf8"),
"Cipher: %s, %s bit, %s" % self.cipher,
"SSL certificate chain:"
]
for n, i in enumerate(self.certchain):
parts.append(" Certificate [%s]" % n)
parts.append("\tSubject: ")
for cn in i.get_subject().get_components():
parts.append("\t\t%s=%s" % (
strutils.always_str(cn[0], "utf8"),
strutils.always_str(cn[1], "utf8"))
)
parts.append("\tIssuer: ")
for cn in i.get_issuer().get_components():
parts.append("\t\t%s=%s" % (
strutils.always_str(cn[0], "utf8"),
strutils.always_str(cn[1], "utf8"))
)
parts.extend(
[
"\tVersion: %s" % i.get_version(),
"\tValidity: %s - %s" % (
strutils.always_str(i.get_notBefore(), "utf8"),
strutils.always_str(i.get_notAfter(), "utf8")
),
"\tSerial: %s" % i.get_serial_number(),
"\tAlgorithm: %s" % strutils.always_str(i.get_signature_algorithm(), "utf8")
]
)
pk = i.get_pubkey()
types = {
OpenSSL.crypto.TYPE_RSA: "RSA",
OpenSSL.crypto.TYPE_DSA: "DSA"
}
t = types.get(pk.type(), "Uknown")
parts.append("\tPubkey: %s bit %s" % (pk.bits(), t))
s = certs.SSLCert(i)
if s.altnames:
parts.append("\tSANs: %s" % " ".join(strutils.always_str(n, "utf8") for n in s.altnames))
return "\n".join(parts)
def __str__(self):
parts = [
"Application Layer Protocol: %s" % strutils.always_str(self.alp, "utf8"),
"Cipher: %s, %s bit, %s" % self.cipher,
"SSL certificate chain:"
]
for n, i in enumerate(self.certchain):
parts.append(" Certificate [%s]" % n)
parts.append("\tSubject: ")
for cn in i.get_subject().get_components():
parts.append("\t\t%s=%s" % (
strutils.always_str(cn[0], "utf8"),
strutils.always_str(cn[1], "utf8"))
)
parts.append("\tIssuer: ")
for cn in i.get_issuer().get_components():
parts.append("\t\t%s=%s" % (
strutils.always_str(cn[0], "utf8"),
strutils.always_str(cn[1], "utf8"))
)
parts.extend(
[
"\tVersion: %s" % i.get_version(),
"\tValidity: %s - %s" % (
strutils.always_str(i.get_notBefore(), "utf8"),
strutils.always_str(i.get_notAfter(), "utf8")
),
"\tSerial: %s" % i.get_serial_number(),
"\tAlgorithm: %s" % strutils.always_str(i.get_signature_algorithm(), "utf8")
]
)
pk = i.get_pubkey()
types = {
OpenSSL.crypto.TYPE_RSA: "RSA",
OpenSSL.crypto.TYPE_DSA: "DSA"
}
t = types.get(pk.type(), "Uknown")
parts.append("\tPubkey: %s bit %s" % (pk.bits(), t))
s = certs.SSLCert(i)
if s.altnames:
parts.append("\tSANs: %s" % " ".join(strutils.always_str(n, "utf8") for n in s.altnames))
return "\n".join(parts)
def make_environ(self, flow, errsoc, **extra):
"""
Raises:
ValueError, if the content-encoding is invalid.
"""
path = strutils.always_str(flow.request.path, "latin-1")
if '?' in path:
path_info, query = strutils.always_str(path, "latin-1").split('?', 1)
else:
path_info = path
query = ''
environ = {
'wsgi.version': (1, 0),
'wsgi.url_scheme': strutils.always_str(flow.request.scheme, "latin-1"),
'wsgi.input': io.BytesIO(flow.request.content or b""),
'wsgi.errors': errsoc,
'wsgi.multithread': True,
'wsgi.multiprocess': False,
'wsgi.run_once': False,
'SERVER_SOFTWARE': self.sversion,
'REQUEST_METHOD': strutils.always_str(flow.request.method, "latin-1"),
'SCRIPT_NAME': '',
'PATH_INFO': urllib.parse.unquote(path_info),
'QUERY_STRING': query,
'CONTENT_TYPE': strutils.always_str(flow.request.headers.get('Content-Type', ''), "latin-1"),
'CONTENT_LENGTH': strutils.always_str(flow.request.headers.get('Content-Length', ''), "latin-1"),
'SERVER_NAME': self.domain,
'SERVER_PORT': str(self.port),
'SERVER_PROTOCOL': strutils.always_str(flow.request.http_version, "latin-1"),
}
environ.update(extra)
if flow.client_conn.address:
environ["REMOTE_ADDR"] = strutils.always_str(flow.client_conn.address.host, "latin-1")
environ["REMOTE_PORT"] = flow.client_conn.address.port
for key, value in flow.request.headers.items():
key = 'HTTP_' + strutils.always_str(key, "latin-1").upper().replace('-', '_')
if key not in ('HTTP_CONTENT_TYPE', 'HTTP_CONTENT_LENGTH'):
environ[key] = value
return environ
def save(self, flows: typing.Sequence[flow.Flow],
cuts: mitmproxy.types.CutSpec,
path: mitmproxy.types.Path) -> None:
"""
Save cuts to file. If there are multiple flows or cuts, the format
is UTF-8 encoded CSV. If there is exactly one row and one column,
the data is written to file as-is, with raw bytes preserved. If the
path is prefixed with a "+", values are appended if there is an
existing file.
"""
append = False
if path.startswith("+"):
append = True
epath = os.path.expanduser(path[1:])
path = mitmproxy.types.Path(epath)
try:
if len(cuts) == 1 and len(flows) == 1:
with open(path, "ab" if append else "wb") as fp:
if fp.tell() > 0:
# We're appending to a file that already exists and has content
fp.write(b"\n")
v = extract(cuts[0], flows[0])
if isinstance(v, bytes):
fp.write(v)
else:
fp.write(v.encode("utf8"))
ctx.log.alert("Saved single cut.")
else:
with open(path,
"a" if append else "w",
newline='',
encoding="utf8") as tfp:
writer = csv.writer(tfp)
for f in flows:
vals = [extract(c, f) for c in cuts]
writer.writerow([
strutils.always_str(x) or "" for x in vals
] # type: ignore
)
ctx.log.alert("Saved %s cuts over %d flows as CSV." %
(len(cuts), len(flows)))
except OSError as e:
ctx.log.error(str(e))
def save(
self,
flows: typing.Sequence[flow.Flow],
cuts: mitmproxy.types.CutSpec,
path: mitmproxy.types.Path
) -> None:
"""
Save cuts to file. If there are multiple flows or cuts, the format
is UTF-8 encoded CSV. If there is exactly one row and one column,
the data is written to file as-is, with raw bytes preserved. If the
path is prefixed with a "+", values are appended if there is an
existing file.
"""
append = False
if path.startswith("+"):
append = True
epath = os.path.expanduser(path[1:])
path = mitmproxy.types.Path(epath)
try:
if len(cuts) == 1 and len(flows) == 1:
with open(path, "ab" if append else "wb") as fp:
if fp.tell() > 0:
# We're appending to a file that already exists and has content
fp.write(b"\n")
v = extract(cuts[0], flows[0])
if isinstance(v, bytes):
fp.write(v)
else:
fp.write(v.encode("utf8"))
ctx.log.alert("Saved single cut.")
else:
with open(path, "a" if append else "w", newline='', encoding="utf8") as fp:
writer = csv.writer(fp)
for f in flows:
vals = [extract(c, f) for c in cuts]
writer.writerow(
[strutils.always_str(x) or "" for x in vals] # type: ignore
)
ctx.log.alert("Saved %s cuts over %d flows as CSV." % (len(cuts), len(flows)))
except IOError as e:
ctx.log.error(str(e))
def url(self, val: Union[str, bytes]) -> None:
val = always_str(val, "utf-8", "surrogateescape")
self.scheme, self.host, self.port, self.path = url.parse(val)
def test_always_str():
with pytest.raises(TypeError):
strutils.always_str(42)
assert strutils.always_str("foo") == "foo"
assert strutils.always_str(b"foo") == "foo"
assert strutils.always_str(None) is None
def conv_conn(conn):
conn["sni"] = strutils.always_str(conn["sni"], "ascii",
"backslashreplace")
conn["alpn"] = conn.pop("alpn_proto_negotiated")
conn["alpn_offers"] = conn["alpn_offers"] or []
conn["cipher_list"] = conn["cipher_list"] or []
def flow_to_json(flow: mitmproxy.flow.Flow) -> dict:
"""
Remove flow message content and cert to save transmission space.
Args:
flow: The original flow.
"""
f = {
"id": flow.id,
"intercepted": flow.intercepted,
"is_replay": flow.is_replay,
"type": flow.type,
"modified": flow.modified(),
"marked": flow.marked,
}
if flow.client_conn:
f["client_conn"] = {
"id": flow.client_conn.id,
"address": flow.client_conn.peername,
"tls_established": flow.client_conn.tls_established,
"timestamp_start": flow.client_conn.timestamp_start,
"timestamp_tls_setup": flow.client_conn.timestamp_tls_setup,
"timestamp_end": flow.client_conn.timestamp_end,
# ideally idna, but we don't want errors
"sni": always_str(flow.client_conn.sni, "ascii", "backslashreplace"),
"cipher_name": flow.client_conn.cipher,
"alpn_proto_negotiated": always_str(flow.client_conn.alpn, "ascii", "backslashreplace"),
"tls_version": flow.client_conn.tls_version,
}
if flow.server_conn:
f["server_conn"] = {
"id": flow.server_conn.id,
"address": flow.server_conn.address,
"ip_address": flow.server_conn.peername,
"source_address": flow.server_conn.sockname,
"tls_established": flow.server_conn.tls_established,
"alpn_proto_negotiated": always_str(flow.server_conn.alpn, "ascii", "backslashreplace"),
"tls_version": flow.server_conn.tls_version,
"timestamp_start": flow.server_conn.timestamp_start,
"timestamp_tcp_setup": flow.server_conn.timestamp_tcp_setup,
"timestamp_tls_setup": flow.server_conn.timestamp_tls_setup,
"timestamp_end": flow.server_conn.timestamp_end,
}
if flow.server_conn.sni is True:
f["server_conn"] = None
else:
# ideally idna, but we don't want errors
f["server_conn"] = always_str(flow.server_conn.sni, "ascii", "backslashreplace")
if flow.error:
f["error"] = flow.error.get_state()
if isinstance(flow, http.HTTPFlow):
content_length: Optional[int]
content_hash: Optional[str]
if flow.request:
if flow.request.raw_content:
content_length = len(flow.request.raw_content)
content_hash = hashlib.sha256(flow.request.raw_content).hexdigest()
else:
content_length = None
content_hash = None
f["request"] = {
"method": flow.request.method,
"scheme": flow.request.scheme,
"host": flow.request.host,
"port": flow.request.port,
"path": flow.request.path,
"http_version": flow.request.http_version,
"headers": tuple(flow.request.headers.items(True)),
"contentLength": content_length,
"contentHash": content_hash,
"timestamp_start": flow.request.timestamp_start,
"timestamp_end": flow.request.timestamp_end,
"is_replay": flow.is_replay == "request", # TODO: remove, use flow.is_replay instead.
"pretty_host": flow.request.pretty_host,
}
if flow.response:
if flow.response.raw_content:
content_length = len(flow.response.raw_content)
content_hash = hashlib.sha256(flow.response.raw_content).hexdigest()
else:
content_length = None
content_hash = None
f["response"] = {
"http_version": flow.response.http_version,
"status_code": flow.response.status_code,
"reason": flow.response.reason,
"headers": tuple(flow.response.headers.items(True)),
"contentLength": content_length,
"contentHash": content_hash,
"timestamp_start": flow.response.timestamp_start,
"timestamp_end": flow.response.timestamp_end,
"is_replay": flow.is_replay == "response", # TODO: remove, use flow.is_replay instead.
}
if flow.response.data.trailers:
f["response"]["trailers"] = tuple(flow.response.data.trailers.items(True))
return f
def _convert_dict_keys(o: Any) -> Any:
if isinstance(o, dict):
return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()}
else:
return o
def test_always_str():
with tutils.raises(TypeError):
strutils.always_str(42)
assert strutils.always_str("foo") == "foo"
assert strutils.always_str(b"foo") == "foo"
assert strutils.always_str(None) is None
def flow_to_json(flow: mitmproxy.flow.Flow) -> dict:
"""
Remove flow message content and cert to save transmission space.
Args:
flow: The original flow.
Sync with web/src/flow.ts.
"""
f = {
"id": flow.id,
"intercepted": flow.intercepted,
"is_replay": flow.is_replay,
"type": flow.type,
"modified": flow.modified(),
"marked": emoji.get(flow.marked, "🔴") if flow.marked else "",
"comment": flow.comment,
}
if flow.client_conn:
f["client_conn"] = {
"id": flow.client_conn.id,
"peername": flow.client_conn.peername,
"sockname": flow.client_conn.sockname,
"tls_established": flow.client_conn.tls_established,
"cert": cert_to_json(flow.client_conn.certificate_list),
"sni": flow.client_conn.sni,
"cipher": flow.client_conn.cipher,
"alpn": always_str(flow.client_conn.alpn, "ascii", "backslashreplace"),
"tls_version": flow.client_conn.tls_version,
"timestamp_start": flow.client_conn.timestamp_start,
"timestamp_tls_setup": flow.client_conn.timestamp_tls_setup,
"timestamp_end": flow.client_conn.timestamp_end,
}
if flow.server_conn:
f["server_conn"] = {
"id": flow.server_conn.id,
"peername": flow.server_conn.peername,
"sockname": flow.server_conn.sockname,
"address": flow.server_conn.address,
"tls_established": flow.server_conn.tls_established,
"cert": cert_to_json(flow.server_conn.certificate_list),
"sni": flow.server_conn.sni,
"cipher": flow.server_conn.cipher,
"alpn": always_str(flow.server_conn.alpn, "ascii", "backslashreplace"),
"tls_version": flow.server_conn.tls_version,
"timestamp_start": flow.server_conn.timestamp_start,
"timestamp_tcp_setup": flow.server_conn.timestamp_tcp_setup,
"timestamp_tls_setup": flow.server_conn.timestamp_tls_setup,
"timestamp_end": flow.server_conn.timestamp_end,
}
if flow.error:
f["error"] = flow.error.get_state()
if isinstance(flow, HTTPFlow):
content_length: Optional[int]
content_hash: Optional[str]
if flow.request.raw_content is not None:
content_length = len(flow.request.raw_content)
content_hash = hashlib.sha256(flow.request.raw_content).hexdigest()
else:
content_length = None
content_hash = None
f["request"] = {
"method": flow.request.method,
"scheme": flow.request.scheme,
"host": flow.request.host,
"port": flow.request.port,
"path": flow.request.path,
"http_version": flow.request.http_version,
"headers": tuple(flow.request.headers.items(True)),
"contentLength": content_length,
"contentHash": content_hash,
"timestamp_start": flow.request.timestamp_start,
"timestamp_end": flow.request.timestamp_end,
"pretty_host": flow.request.pretty_host,
}
if flow.response:
if flow.response.raw_content is not None:
content_length = len(flow.response.raw_content)
content_hash = hashlib.sha256(flow.response.raw_content).hexdigest()
else:
content_length = None
content_hash = None
f["response"] = {
"http_version": flow.response.http_version,
"status_code": flow.response.status_code,
"reason": flow.response.reason,
"headers": tuple(flow.response.headers.items(True)),
"contentLength": content_length,
"contentHash": content_hash,
"timestamp_start": flow.response.timestamp_start,
"timestamp_end": flow.response.timestamp_end,
}
if flow.response.data.trailers:
f["response"]["trailers"] = tuple(flow.response.data.trailers.items(True))
if flow.websocket:
f["websocket"] = {
"messages_meta": {
"contentLength": sum(len(x.content) for x in flow.websocket.messages),
"count": len(flow.websocket.messages),
"timestamp_last": flow.websocket.messages[-1].timestamp if flow.websocket.messages else None,
},
"closed_by_client": flow.websocket.closed_by_client,
"close_code": flow.websocket.close_code,
"close_reason": flow.websocket.close_reason,
"timestamp_end": flow.websocket.timestamp_end,
}
elif isinstance(flow, TCPFlow):
f["messages_meta"] = {
"contentLength": sum(len(x.content) for x in flow.messages),
"count": len(flow.messages),
"timestamp_last": flow.messages[-1].timestamp if flow.messages else None,
}
return f