def parse_overlay_outstats(entry):
if entry.source != "wifi_ol":
return []
# data format:
# OUT-STATS host=[hostname] peer=[hostname] queued-pkts=X queued-ctrl=X
# queued-bytes=X drop-pkts=X drop-ctrl=X drop-bytes=X out-all=X send-bytes=X
(head, _, tail) = entry.data.partition(" ")
if head != "OUT-STATS":
return []
vals = dictify(tail)
for field in ["host", "peer", "queued-pkts", "queued-ctrl", "queued-bytes",
"drop-pkts", "drop-ctrl", "drop-bytes", "out-all", "send-bytes"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
host = argosroutes.to_hostname(vals["host"])
return [ (host, vals["peer"], int(vals["queued-pkts"]),
int(vals["queued-ctrl"]), int(vals["queued-bytes"]),
int(vals["drop-pkts"]), int(vals["drop-ctrl"]),
int(vals["drop-bytes"]), int(vals["out-all"]),
int(vals["send-bytes"]) ) ]
def parse_entry(self, entry):
try:
if entry.source == "ol_in_proxy":
(head, _, tail) = entry.data.partition(" ")
if head == "STATS":
self.parse_wifi_ol_inproxy_stats(entry.datetime, dictify(tail))
elif entry.source == "pcap":
(head, _, tail) = entry.data.partition(" ")
if head == "STATS":
self.parse_pcap_stats(entry.datetime, dictify(tail))
elif entry.source == "queries":
(head, _, tail) = entry.data.partition(" ")
if head == "STATS":
self.parse_query_stats(entry.datetime, dictify(tail))
elif entry.source == "to_server":
(head, _, tail) = entry.data.partition(" ")
if head == "STATS":
self.parse_toserver_stats(entry.datetime, dictify(tail))
elif head == "OUT-STATS":
self.parse_toserver_out_stats(entry.datetime, dictify(tail))
# special case: all 'wifi_ol_xxx' sources count as the same source
# a quick test shows slicing is faster than startswith()
elif entry.source[:7] == "wifi_ol":
(head, _, tail) = entry.data.partition(" ")
if head == "STATS":
# faster version of "if entry.source == 'wifi_ol'"
if len(entry.source) == 7:
# this is from a node...
self.parse_wifi_ol_sniffer_stats(entry.datetime, dictify(tail))
else:
# this is from the server...
self.parse_wifi_ol_server_stats(entry.datetime, dictify(tail), entry.source[8:])
elif head == "IN-STATS":
self.parse_wifi_ol_in_stats(entry.datetime, dictify(tail))
elif head == "OUT-STATS":
self.parse_wifi_ol_out_stats(entry.datetime, dictify(tail))
except KeyError, e:
raise ValueError("field '%s' not found" % e.args[0])
def teams_stats(date):
teams = dictify('teams {}'.format(date), 'teams')
players = dictify('players {}'.format(date), 'players')
d = {} #dictionary holding all the team stats
count = 0
#Recall minitues is 5 index
for team in teams.items():
name = team[0]
players_on_team = team[1]
#We have our team name and the players on that team
mins = []
for p in players_on_team:
stats = players[p]
mins.append(float(stats[5]))
mins.sort(reverse=True) # sorted list of minutes played
cutoff = mins[6] #abitrary cutoff. Not gonna count the people who dont play that much, only top 7
for p in players_on_team:
stats = players[p]
playing_time = float(stats[5])
#remove the non numerical data
stats.pop(0)
stats.pop(1)
stats.pop(3)
if playing_time >= cutoff:
if name in d:
d[name] = add_arrays(d[name], stats)
else:
d[name] = stats
return d
def main():
s = 'foo=1 bar="chose" x p="aweomse sdsd" zal=4 foo=9'
last = time.time()
while 1:
now = time.time()
if now > last + 1:
res = resource.getrusage(resource.RUSAGE_SELF)
maxrss = res.ru_maxrss
mbytes = maxrss*resource.getpagesize()/float(1024*1024)
print "maxrss: %u pages (%u MB)" % (maxrss, mbytes)
last = now
else:
# do some work!
for i in range(100*1000):
vals = dictify.dictify(s)
if vals is None:
raise ValueError("dictify returned None")
def parse_overlay_instats(entry):
if entry.source != "wifi_ol":
return []
# data format:
# IN-STATS host=[hostname] peer=[hostname] in-pkts=X in-ctrl=X in-bytes=X
(head, _, tail) = entry.data.partition(" ")
if head != "IN-STATS":
return []
vals = dictify(tail)
for field in ["host", "peer", "in-pkts", "in-ctrl", "in-bytes"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
host = argosroutes.to_hostname(vals["host"])
return [ (host, vals["peer"], int(vals["in-pkts"]),
int(vals["in-ctrl"]), int(vals["in-bytes"])) ]
def parse_server_traffic(entry):
if entry.source not in ["net_proxy", "to_server"]:
return []
# data format:
# STATS host=[hostname] port=[port] count=X bytes=X
(head, _, tail) = entry.data.partition(" ")
if head != "STATS":
return []
vals = dictify(tail)
for field in ["host", "bytes", "count"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
bytes = estimate_network_bytes(int(vals["bytes"]), int(vals["count"]))
host = argosroutes.to_hostname(vals["host"])
return [(host, bytes)]
def parse_merge_rate(entry):
if entry.source != "wifi_merge":
return []
# data format:
# <FILTER> host=<hostname> <other fields>
(head, _, tail) = entry.data.partition(" ")
if head != "STATS":
return []
vals = dictify(tail)
for field in ["host", "avg_merge", "out_merges"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
if int(vals["out_merges"]) == 0:
return []
host = argosroutes.to_hostname(vals["host"])
return [(host, float(vals["avg_merge"]))]
def parse_basic_line(entry, source_filter, head_filter, key, typecast=int, strict=True):
if source_filter is not None:
# LEGACY SUPPORT
# in old versions, the 'pcap' source was named 'script', so if
# source_filter="pcap", then check for either one
if source_filter == "pcap":
if entry.source not in [source_filter, "script"]:
return []
else:
# normal check
if entry.source != source_filter:
return []
# data format:
# <FILTER> host=<hostname> <other fields>
(head, _, tail) = entry.data.partition(" ")
if head != head_filter:
return []
vals = dictify(tail)
# also support old-style names that used underscores instead of dashes
for k, v in vals.iteritems():
if "_" in k:
vals[k.replace("_", "-")] = v
del vals[k]
for field in ["host", key]:
if field not in vals:
if strict:
print "bad line (no '%s' field): %s" % (field, entry)
return []
val = typecast(vals[key])
host = argosroutes.to_hostname(vals["host"])
return [(host, val)]
def parse_net_links_usage(entry):
# list of (link, (overlay-bytes, mergestream-bytes, rawstream-bytes)) tuples
data = []
(head, _, tail) = entry.data.partition(" ")
if entry.source in ["net_proxy", "to_server"]:
# data format:
# STATS host=[hostname] port=[port] count=X bytes=X
if head == "STATS":
# this tells us the number of bytes processed by the query running
# on this node
vals = dictify(tail)
for field in ["host", "port", "bytes", "count"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
# the net_proxy logs multiple different lines, differentiated by the
# 'port' field:
# 'log' tracks log messages from the core system (not from queries)
# 'stream' tracks packets streamed directly to the server
# 'qX' tracks all NetworkProxy usage by query X
# 'on_wire' tracks the (aggregate) data actually sent on the socket
# - note that this is post-compression whereas all of the other
# counts are pre-compression
if vals["port"] != "on_wire":
return []
bytes = int(vals["bytes"])
host = argosroutes.to_hostname(vals["host"])
for link in argosroutes.walk_route(host, "citysense"):
data.append((link, (0, bytes, 0)))
elif entry.source == "script":
# data format:
# STATS host=[hostname] kern_recv=X kern_drop=X pre_capt_pkts=X pre_capt_bytes=X
# post_capt_pkts=X post_capt_bytes=X filter_pkts=X filter_bytes=X
if head == "STATS":
vals = dictify(tail)
for field in ["host", "post_capt_bytes", "post_capt_pkts"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
# 'post_capt_bytes' field is bytes of packets, not network bytes
bytes = estimate_network_bytes(int(vals["post_capt_bytes"]),
int(vals["post_capt_pkts"]))
host = argosroutes.to_hostname(vals["host"])
# 'link' is a (host,host) tuple
for link in argosroutes.walk_route(host, "citysense"):
data.append((link, (0, 0, bytes)))
elif entry.source == "wifi_ol":
# data format:
# NET-USAGE src=[ip] dst=[ip] capt_pkts=X capt_bytes=X | (repeat...)
if head == "NET-USAGE":
parts = tail.split("|")
for part in parts:
vals = dictify(part)
for field in ["capt_bytes", "src", "dst"]:
if field not in vals:
print "bad line (no '%s' field): %s" % (field, entry)
return []
# 'capt_bytes' field is actual network bytes, not just packet bytes
bytes = int(vals["capt_bytes"])
src = argosroutes.to_hostname(vals["src"])
dst = argosroutes.to_hostname(vals["dst"])
if src == dst:
# these are "self-routed" bytes which do not traverse the
# network at all
continue
if bytes == 0:
continue
if argosroutes.to_hostname(src) == "citysense":
# these should all be control messages, so we just ignore them
continue
# 'link' is a (host,host) tuple
for link in argosroutes.walk_route(src, dst):
data.append((link, (bytes, 0, 0)))
return data
def main():
li = [("a=1", {"a": "1"}),
("a=\"1\"", {"a": "1"}),
("a='1'", {"a": "1"}),
("a=1 ", {"a": "1"}),
("a=\"1\" ", {"a": "1"}),
("a='1' ", {"a": "1"}),
("a= 1", {"a": "1"}),
("a= \"1\"", {"a": "1"}),
("a= '1'", {"a": "1"}),
("a = 1", {"a": "1"}),
("a = \"1\"", {"a": "1"}),
("a = '1'", {"a": "1"}),
("a='1 '", {"a": "1 "}),
("a=' 1'", {"a": " 1"}),
("a= ' 1 '", {"a": " 1 "}),
("a =' 1 '", {"a": " 1 "}),
("a = ' 1 ' ", {"a": " 1 "}),
("a='1\"'", {"a": "1\""}),
("a=\"1'\"", {"a": "1'"}),
("a=\"1'2'3\"", {"a": "1'2'3"}),
("a='\"123\"'", {"a": "\"123\""}),
("a", {"a": None}),
("a ", {"a": None}),
(" a", {"a": None}),
(" a ", {"a": None}),
("a=", None),
(" a=", None),
(" a= ", None),
("a=\"", None),
("a=\"b", None),
("a='b", None),
("a=\" ", None),
("a=\"b ", None),
("a='b ", None),
("a=\" ", None),
("a=\"b'bb", None),
("a='b\"bb", None),
("a=1 b=2", {"a": "1", "b": "2"}),
("a=1 b= 2", {"a": "1", "b": "2"}),
("a=1 b =2", {"a": "1", "b": "2"}),
("a=1 b = 2", {"a": "1", "b": "2"}),
("a=1 b=2 ", {"a": "1", "b": "2"}),
("a=1 b = 2 ", {"a": "1", "b": "2"}),
(" a = 1 b = 2 ", {"a": "1", "b": "2"}),
("a=1 b='2'", {"a": "1", "b": "2"}),
("a=1 b=\"2\"", {"a": "1", "b": "2"}),
("a=1 b=\"2\" ", {"a": "1", "b": "2"}),
("a=1 b = \"2\" ", {"a": "1", "b": "2"}),
("a=1 b", {"a": "1", "b": None}),
("a=1 b ", {"a": "1", "b": None}),
("a=1 b ", {"a": "1", "b": None}),
("a=1 b=2", {"a": "1", "b": "2"}),
("a=1 b=2 ", {"a": "1", "b": "2"}),
("a=1 b = 2", {"a": "1", "b": "2"}),
("a=1 b='2'", {"a": "1", "b": "2"}),
("a=1 b= \"2\" ", {"a": "1", "b": "2"}),
("a=1 b=", None),
("a=1 b='2", None),
("a=1 b=\"2", None),
("a=1 b= '2", None),
("a=1 b='2 ", None),
("a=1 b= '2 ", None),
("a=1 b= '2 ' ", {"a": "1", "b": "2 "}),
("a=1 b=2 c=3", {"a": "1", "b": "2", "c": "3"}),
("a=1 b=2 c=3 ", {"a": "1", "b": "2", "c": "3"}),
("a=1 b=2 c='3'", {"a": "1", "b": "2", "c": "3"}),
("x a=1 b=2 c=3", {"x": None, "a": "1", "b": "2", "c": "3"}),
("a=1 x b=2 c=3", {"x": None, "a": "1", "b": "2", "c": "3"}),
("a= 1 x b=2 c=3", {"x": None, "a": "1", "b": "2", "c": "3"}),
("a= 1 b=2 c=3 x", {"x": None, "a": "1", "b": "2", "c": "3"}),
("a= 1 b=2 c=3 x x x", {"x": None, "a": "1", "b": "2", "c": "3"}),
("a= 1 x b=2 c=3 x b=4 x c=5 x x", {"x": None, "a": "1", "b": "4", "c": "5"})
]
for (s, expected) in li:
try:
rv = dictify(s)
except ParseError, e:
if expected is not None:
print "test failed! erroneous ParseError (%s)" % str(e)
print "arg: [%s]" % s
sys.exit(1)
except Exception, e:
print "unexpected exception: %s" % repr(e)
print "arg: [%s]" % s
print
raise
