def parse_overlay_outstats(entry): if entry.source != "wifi_ol": return [] # data format: # OUT-STATS host=[hostname] peer=[hostname] queued-pkts=X queued-ctrl=X # queued-bytes=X drop-pkts=X drop-ctrl=X drop-bytes=X out-all=X send-bytes=X (head, _, tail) = entry.data.partition(" ") if head != "OUT-STATS": return [] vals = dictify(tail) for field in ["host", "peer", "queued-pkts", "queued-ctrl", "queued-bytes", "drop-pkts", "drop-ctrl", "drop-bytes", "out-all", "send-bytes"]: if field not in vals: print "bad line (no '%s' field): %s" % (field, entry) return [] host = argosroutes.to_hostname(vals["host"]) return [ (host, vals["peer"], int(vals["queued-pkts"]), int(vals["queued-ctrl"]), int(vals["queued-bytes"]), int(vals["drop-pkts"]), int(vals["drop-ctrl"]), int(vals["drop-bytes"]), int(vals["out-all"]), int(vals["send-bytes"]) ) ]
def parse_entry(self, entry): try: if entry.source == "ol_in_proxy": (head, _, tail) = entry.data.partition(" ") if head == "STATS": self.parse_wifi_ol_inproxy_stats(entry.datetime, dictify(tail)) elif entry.source == "pcap": (head, _, tail) = entry.data.partition(" ") if head == "STATS": self.parse_pcap_stats(entry.datetime, dictify(tail)) elif entry.source == "queries": (head, _, tail) = entry.data.partition(" ") if head == "STATS": self.parse_query_stats(entry.datetime, dictify(tail)) elif entry.source == "to_server": (head, _, tail) = entry.data.partition(" ") if head == "STATS": self.parse_toserver_stats(entry.datetime, dictify(tail)) elif head == "OUT-STATS": self.parse_toserver_out_stats(entry.datetime, dictify(tail)) # special case: all 'wifi_ol_xxx' sources count as the same source # a quick test shows slicing is faster than startswith() elif entry.source[:7] == "wifi_ol": (head, _, tail) = entry.data.partition(" ") if head == "STATS": # faster version of "if entry.source == 'wifi_ol'" if len(entry.source) == 7: # this is from a node... self.parse_wifi_ol_sniffer_stats(entry.datetime, dictify(tail)) else: # this is from the server... self.parse_wifi_ol_server_stats(entry.datetime, dictify(tail), entry.source[8:]) elif head == "IN-STATS": self.parse_wifi_ol_in_stats(entry.datetime, dictify(tail)) elif head == "OUT-STATS": self.parse_wifi_ol_out_stats(entry.datetime, dictify(tail)) except KeyError, e: raise ValueError("field '%s' not found" % e.args[0])
def teams_stats(date): teams = dictify('teams {}'.format(date), 'teams') players = dictify('players {}'.format(date), 'players') d = {} #dictionary holding all the team stats count = 0 #Recall minitues is 5 index for team in teams.items(): name = team[0] players_on_team = team[1] #We have our team name and the players on that team mins = [] for p in players_on_team: stats = players[p] mins.append(float(stats[5])) mins.sort(reverse=True) # sorted list of minutes played cutoff = mins[6] #abitrary cutoff. Not gonna count the people who dont play that much, only top 7 for p in players_on_team: stats = players[p] playing_time = float(stats[5]) #remove the non numerical data stats.pop(0) stats.pop(1) stats.pop(3) if playing_time >= cutoff: if name in d: d[name] = add_arrays(d[name], stats) else: d[name] = stats return d
def main(): s = 'foo=1 bar="chose" x p="aweomse sdsd" zal=4 foo=9' last = time.time() while 1: now = time.time() if now > last + 1: res = resource.getrusage(resource.RUSAGE_SELF) maxrss = res.ru_maxrss mbytes = maxrss*resource.getpagesize()/float(1024*1024) print "maxrss: %u pages (%u MB)" % (maxrss, mbytes) last = now else: # do some work! for i in range(100*1000): vals = dictify.dictify(s) if vals is None: raise ValueError("dictify returned None")
def parse_overlay_instats(entry): if entry.source != "wifi_ol": return [] # data format: # IN-STATS host=[hostname] peer=[hostname] in-pkts=X in-ctrl=X in-bytes=X (head, _, tail) = entry.data.partition(" ") if head != "IN-STATS": return [] vals = dictify(tail) for field in ["host", "peer", "in-pkts", "in-ctrl", "in-bytes"]: if field not in vals: print "bad line (no '%s' field): %s" % (field, entry) return [] host = argosroutes.to_hostname(vals["host"]) return [ (host, vals["peer"], int(vals["in-pkts"]), int(vals["in-ctrl"]), int(vals["in-bytes"])) ]
def parse_server_traffic(entry): if entry.source not in ["net_proxy", "to_server"]: return [] # data format: # STATS host=[hostname] port=[port] count=X bytes=X (head, _, tail) = entry.data.partition(" ") if head != "STATS": return [] vals = dictify(tail) for field in ["host", "bytes", "count"]: if field not in vals: print "bad line (no '%s' field): %s" % (field, entry) return [] bytes = estimate_network_bytes(int(vals["bytes"]), int(vals["count"])) host = argosroutes.to_hostname(vals["host"]) return [(host, bytes)]
def parse_merge_rate(entry): if entry.source != "wifi_merge": return [] # data format: # <FILTER> host=<hostname> <other fields> (head, _, tail) = entry.data.partition(" ") if head != "STATS": return [] vals = dictify(tail) for field in ["host", "avg_merge", "out_merges"]: if field not in vals: print "bad line (no '%s' field): %s" % (field, entry) return [] if int(vals["out_merges"]) == 0: return [] host = argosroutes.to_hostname(vals["host"]) return [(host, float(vals["avg_merge"]))]
def parse_basic_line(entry, source_filter, head_filter, key, typecast=int, strict=True): if source_filter is not None: # LEGACY SUPPORT # in old versions, the 'pcap' source was named 'script', so if # source_filter="pcap", then check for either one if source_filter == "pcap": if entry.source not in [source_filter, "script"]: return [] else: # normal check if entry.source != source_filter: return [] # data format: # <FILTER> host=<hostname> <other fields> (head, _, tail) = entry.data.partition(" ") if head != head_filter: return [] vals = dictify(tail) # also support old-style names that used underscores instead of dashes for k, v in vals.iteritems(): if "_" in k: vals[k.replace("_", "-")] = v del vals[k] for field in ["host", key]: if field not in vals: if strict: print "bad line (no '%s' field): %s" % (field, entry) return [] val = typecast(vals[key]) host = argosroutes.to_hostname(vals["host"]) return [(host, val)]
def parse_net_links_usage(entry): # list of (link, (overlay-bytes, mergestream-bytes, rawstream-bytes)) tuples data = [] (head, _, tail) = entry.data.partition(" ") if entry.source in ["net_proxy", "to_server"]: # data format: # STATS host=[hostname] port=[port] count=X bytes=X if head == "STATS": # this tells us the number of bytes processed by the query running # on this node vals = dictify(tail) for field in ["host", "port", "bytes", "count"]: if field not in vals: print "bad line (no '%s' field): %s" % (field, entry) return [] # the net_proxy logs multiple different lines, differentiated by the # 'port' field: # 'log' tracks log messages from the core system (not from queries) # 'stream' tracks packets streamed directly to the server # 'qX' tracks all NetworkProxy usage by query X # 'on_wire' tracks the (aggregate) data actually sent on the socket # - note that this is post-compression whereas all of the other # counts are pre-compression if vals["port"] != "on_wire": return [] bytes = int(vals["bytes"]) host = argosroutes.to_hostname(vals["host"]) for link in argosroutes.walk_route(host, "citysense"): data.append((link, (0, bytes, 0))) elif entry.source == "script": # data format: # STATS host=[hostname] kern_recv=X kern_drop=X pre_capt_pkts=X pre_capt_bytes=X # post_capt_pkts=X post_capt_bytes=X filter_pkts=X filter_bytes=X if head == "STATS": vals = dictify(tail) for field in ["host", "post_capt_bytes", "post_capt_pkts"]: if field not in vals: print "bad line (no '%s' field): %s" % (field, entry) return [] # 'post_capt_bytes' field is bytes of packets, not network bytes bytes = estimate_network_bytes(int(vals["post_capt_bytes"]), int(vals["post_capt_pkts"])) host = argosroutes.to_hostname(vals["host"]) # 'link' is a (host,host) tuple for link in argosroutes.walk_route(host, "citysense"): data.append((link, (0, 0, bytes))) elif entry.source == "wifi_ol": # data format: # NET-USAGE src=[ip] dst=[ip] capt_pkts=X capt_bytes=X | (repeat...) if head == "NET-USAGE": parts = tail.split("|") for part in parts: vals = dictify(part) for field in ["capt_bytes", "src", "dst"]: if field not in vals: print "bad line (no '%s' field): %s" % (field, entry) return [] # 'capt_bytes' field is actual network bytes, not just packet bytes bytes = int(vals["capt_bytes"]) src = argosroutes.to_hostname(vals["src"]) dst = argosroutes.to_hostname(vals["dst"]) if src == dst: # these are "self-routed" bytes which do not traverse the # network at all continue if bytes == 0: continue if argosroutes.to_hostname(src) == "citysense": # these should all be control messages, so we just ignore them continue # 'link' is a (host,host) tuple for link in argosroutes.walk_route(src, dst): data.append((link, (bytes, 0, 0))) return data
def main(): li = [("a=1", {"a": "1"}), ("a=\"1\"", {"a": "1"}), ("a='1'", {"a": "1"}), ("a=1 ", {"a": "1"}), ("a=\"1\" ", {"a": "1"}), ("a='1' ", {"a": "1"}), ("a= 1", {"a": "1"}), ("a= \"1\"", {"a": "1"}), ("a= '1'", {"a": "1"}), ("a = 1", {"a": "1"}), ("a = \"1\"", {"a": "1"}), ("a = '1'", {"a": "1"}), ("a='1 '", {"a": "1 "}), ("a=' 1'", {"a": " 1"}), ("a= ' 1 '", {"a": " 1 "}), ("a =' 1 '", {"a": " 1 "}), ("a = ' 1 ' ", {"a": " 1 "}), ("a='1\"'", {"a": "1\""}), ("a=\"1'\"", {"a": "1'"}), ("a=\"1'2'3\"", {"a": "1'2'3"}), ("a='\"123\"'", {"a": "\"123\""}), ("a", {"a": None}), ("a ", {"a": None}), (" a", {"a": None}), (" a ", {"a": None}), ("a=", None), (" a=", None), (" a= ", None), ("a=\"", None), ("a=\"b", None), ("a='b", None), ("a=\" ", None), ("a=\"b ", None), ("a='b ", None), ("a=\" ", None), ("a=\"b'bb", None), ("a='b\"bb", None), ("a=1 b=2", {"a": "1", "b": "2"}), ("a=1 b= 2", {"a": "1", "b": "2"}), ("a=1 b =2", {"a": "1", "b": "2"}), ("a=1 b = 2", {"a": "1", "b": "2"}), ("a=1 b=2 ", {"a": "1", "b": "2"}), ("a=1 b = 2 ", {"a": "1", "b": "2"}), (" a = 1 b = 2 ", {"a": "1", "b": "2"}), ("a=1 b='2'", {"a": "1", "b": "2"}), ("a=1 b=\"2\"", {"a": "1", "b": "2"}), ("a=1 b=\"2\" ", {"a": "1", "b": "2"}), ("a=1 b = \"2\" ", {"a": "1", "b": "2"}), ("a=1 b", {"a": "1", "b": None}), ("a=1 b ", {"a": "1", "b": None}), ("a=1 b ", {"a": "1", "b": None}), ("a=1 b=2", {"a": "1", "b": "2"}), ("a=1 b=2 ", {"a": "1", "b": "2"}), ("a=1 b = 2", {"a": "1", "b": "2"}), ("a=1 b='2'", {"a": "1", "b": "2"}), ("a=1 b= \"2\" ", {"a": "1", "b": "2"}), ("a=1 b=", None), ("a=1 b='2", None), ("a=1 b=\"2", None), ("a=1 b= '2", None), ("a=1 b='2 ", None), ("a=1 b= '2 ", None), ("a=1 b= '2 ' ", {"a": "1", "b": "2 "}), ("a=1 b=2 c=3", {"a": "1", "b": "2", "c": "3"}), ("a=1 b=2 c=3 ", {"a": "1", "b": "2", "c": "3"}), ("a=1 b=2 c='3'", {"a": "1", "b": "2", "c": "3"}), ("x a=1 b=2 c=3", {"x": None, "a": "1", "b": "2", "c": "3"}), ("a=1 x b=2 c=3", {"x": None, "a": "1", "b": "2", "c": "3"}), ("a= 1 x b=2 c=3", {"x": None, "a": "1", "b": "2", "c": "3"}), ("a= 1 b=2 c=3 x", {"x": None, "a": "1", "b": "2", "c": "3"}), ("a= 1 b=2 c=3 x x x", {"x": None, "a": "1", "b": "2", "c": "3"}), ("a= 1 x b=2 c=3 x b=4 x c=5 x x", {"x": None, "a": "1", "b": "4", "c": "5"}) ] for (s, expected) in li: try: rv = dictify(s) except ParseError, e: if expected is not None: print "test failed! erroneous ParseError (%s)" % str(e) print "arg: [%s]" % s sys.exit(1) except Exception, e: print "unexpected exception: %s" % repr(e) print "arg: [%s]" % s print raise