def parse_overlay_outstats(entry):
if entry.source != "wifi_ol":
return []
# data format:
# OUT-STATS host=[hostname] peer=[hostname] queued-pkts=X queued-ctrl=X
# queued-bytes=X drop-pkts=X drop-ctrl=X drop-bytes=X out-all=X send-bytes=X
(head, _, tail) = entry.data.partition(" ")
if head != "OUT-STATS":
return []
vals = dictify(tail)
for field in ["host", "peer", "queued-pkts", "queued-ctrl", "queued-bytes",
"drop-pkts", "drop-ctrl", "drop-bytes", "out-all", "send-bytes"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
host = argosroutes.to_hostname(vals["host"])
return [ (host, vals["peer"], int(vals["queued-pkts"]),
int(vals["queued-ctrl"]), int(vals["queued-bytes"]),
int(vals["drop-pkts"]), int(vals["drop-ctrl"]),
int(vals["drop-bytes"]), int(vals["out-all"]),
int(vals["send-bytes"]) ) ]
def parse_overlay_instats(entry):
if entry.source != "wifi_ol":
return []
# data format:
# IN-STATS host=[hostname] peer=[hostname] in-pkts=X in-ctrl=X in-bytes=X
(head, _, tail) = entry.data.partition(" ")
if head != "IN-STATS":
return []
vals = dictify(tail)
for field in ["host", "peer", "in-pkts", "in-ctrl", "in-bytes"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
host = argosroutes.to_hostname(vals["host"])
return [ (host, vals["peer"], int(vals["in-pkts"]),
int(vals["in-ctrl"]), int(vals["in-bytes"])) ]
def parse_server_traffic(entry):
if entry.source not in ["net_proxy", "to_server"]:
return []
# data format:
# STATS host=[hostname] port=[port] count=X bytes=X
(head, _, tail) = entry.data.partition(" ")
if head != "STATS":
return []
vals = dictify(tail)
for field in ["host", "bytes", "count"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
bytes = estimate_network_bytes(int(vals["bytes"]), int(vals["count"]))
host = argosroutes.to_hostname(vals["host"])
return [(host, bytes)]
def parse_merge_rate(entry):
if entry.source != "wifi_merge":
return []
# data format:
# <FILTER> host=<hostname> <other fields>
(head, _, tail) = entry.data.partition(" ")
if head != "STATS":
return []
vals = dictify(tail)
for field in ["host", "avg_merge", "out_merges"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
if int(vals["out_merges"]) == 0:
return []
host = argosroutes.to_hostname(vals["host"])
return [(host, float(vals["avg_merge"]))]
def parse_basic_line(entry, source_filter, head_filter, key, typecast=int, strict=True):
if source_filter is not None:
# LEGACY SUPPORT
# in old versions, the 'pcap' source was named 'script', so if
# source_filter="pcap", then check for either one
if source_filter == "pcap":
if entry.source not in [source_filter, "script"]:
return []
else:
# normal check
if entry.source != source_filter:
return []
# data format:
# <FILTER> host=<hostname> <other fields>
(head, _, tail) = entry.data.partition(" ")
if head != head_filter:
return []
vals = dictify(tail)
# also support old-style names that used underscores instead of dashes
for k, v in vals.iteritems():
if "_" in k:
vals[k.replace("_", "-")] = v
del vals[k]
for field in ["host", key]:
if field not in vals:
if strict:
print "bad line (no '%s' field): %s" % (field, entry)
return []
val = typecast(vals[key])
host = argosroutes.to_hostname(vals["host"])
return [(host, val)]
def parse_net_links_usage(entry):
# list of (link, (overlay-bytes, mergestream-bytes, rawstream-bytes)) tuples
data = []
(head, _, tail) = entry.data.partition(" ")
if entry.source in ["net_proxy", "to_server"]:
# data format:
# STATS host=[hostname] port=[port] count=X bytes=X
if head == "STATS":
# this tells us the number of bytes processed by the query running
# on this node
vals = dictify(tail)
for field in ["host", "port", "bytes", "count"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
# the net_proxy logs multiple different lines, differentiated by the
# 'port' field:
# 'log' tracks log messages from the core system (not from queries)
# 'stream' tracks packets streamed directly to the server
# 'qX' tracks all NetworkProxy usage by query X
# 'on_wire' tracks the (aggregate) data actually sent on the socket
# - note that this is post-compression whereas all of the other
# counts are pre-compression
if vals["port"] != "on_wire":
return []
bytes = int(vals["bytes"])
host = argosroutes.to_hostname(vals["host"])
for link in argosroutes.walk_route(host, "citysense"):
data.append((link, (0, bytes, 0)))
elif entry.source == "script":
# data format:
# STATS host=[hostname] kern_recv=X kern_drop=X pre_capt_pkts=X pre_capt_bytes=X
# post_capt_pkts=X post_capt_bytes=X filter_pkts=X filter_bytes=X
if head == "STATS":
vals = dictify(tail)
for field in ["host", "post_capt_bytes", "post_capt_pkts"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
# 'post_capt_bytes' field is bytes of packets, not network bytes
bytes = estimate_network_bytes(int(vals["post_capt_bytes"]),
int(vals["post_capt_pkts"]))
host = argosroutes.to_hostname(vals["host"])
# 'link' is a (host,host) tuple
for link in argosroutes.walk_route(host, "citysense"):
data.append((link, (0, 0, bytes)))
elif entry.source == "wifi_ol":
# data format:
# NET-USAGE src=[ip] dst=[ip] capt_pkts=X capt_bytes=X | (repeat...)
if head == "NET-USAGE":
parts = tail.split("|")
for part in parts:
vals = dictify(part)
for field in ["capt_bytes", "src", "dst"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
# 'capt_bytes' field is actual network bytes, not just packet bytes
bytes = int(vals["capt_bytes"])
src = argosroutes.to_hostname(vals["src"])
dst = argosroutes.to_hostname(vals["dst"])
if src == dst:
# these are "self-routed" bytes which do not traverse the
# network at all
continue
if bytes == 0:
continue
if argosroutes.to_hostname(src) == "citysense":
# these should all be control messages, so we just ignore them
continue
# 'link' is a (host,host) tuple
for link in argosroutes.walk_route(src, dst):
data.append((link, (bytes, 0, 0)))
return data
