def loadfile(self, path):
ln = 0
with open(path, "r") as f:
for line in f.readlines():
ln += 1
l = line.lstrip()
if not l or l[0] == '#':
continue
if l[0] == '%':
self.loadfile(l[1:])
continue
equalpos = l.find("=")
if equalpos < 0:
continue
key = l[:equalpos]
if len(key) < 4:
continue
val = l[equalpos + 1:]
try:
if not self.set(key, val, callchanged=False):
klog.e("BADCONF: %s:%d: '%s'" % (path, ln, line))
except:
traceback.print_exc()
klog.e("BADCONF: %s:%d: '%s'" % (path, ln, line))
self._conf_changed()
def onflow(self, routerip, rec):
dic, msg = nfrp.rec2dic(routerip, rec)
nfp_recs.inc()
if dic:
nfp_recs_ok.inc()
nfrp.savedic(dic)
else:
nfp_recs_ng.inc()
klog.e("Error from rec2dic: ", msg)
def unload(self, host):
hashkeys = self.readyRouters.get(host)
if not hashkeys:
klog.e("unload router %s failed, it not loaded yet" % host)
return
klog.d("unload router: %s" % host)
for hashkey in hashkeys:
try:
del self.portInfos[hashkey]
except:
pass
try:
del self.readyRouters[host]
except:
pass
def get_vlink_flow(uid):
flows = []
show = {"_id": 0, "bps": 1, "time.log": 1, "addr.src": 1, "addr.dst": 1}
# vlink.sport, vlink.dport => db.ports.ui
vlink = getvlink(uid)
if not vlink:
klog.e("NotFound: vlink: uid:", uid)
return []
klog.d(varfmt(vlink, "vlink"))
sportip = vlink.get("sportip")
dportip = vlink.get("dportip")
src_equip = equip_fr_port_ip(sportip) or equip_fr_port_ip(
vlink["sloopbackip"])
dst_equip = equip_fr_port_ip(dportip) or equip_fr_port_ip(
vlink["dloopbackip"])
if not src_equip:
klog.e("NotFound: src_equip: sportip:", sportip)
return []
next_hop_uid = dst_equip.get("uid", "<Null>")
match = {"loopback.cur": src_equip["ip_str"], "port.nexthop": dportip}
klog.d(varfmt(match, "db.logs.find: match"))
logs = db.logs.find(match, show)
for log in logs:
try:
timediff = time.time() - log["time"]["log"]
if timediff > conf.DB_LOG_AGE_SKIP:
continue
_flow = "(%s) >>> (%s)@(%s) >>> (%s)@(%s) >>> (%s)" % (
log["addr"]["src"], sportip, src_equip["ip_str"], dportip,
dst_equip["ip_str"], log["addr"]["dst"])
flow = {
"_flow": _flow,
"bps": log["bps"],
"src": log["addr"]["src"],
"dst": log["addr"]["dst"],
"uid": uid,
"next_hop_uid": next_hop_uid
}
flows.append(flow)
except:
klog.e(traceback.format_exc())
klog.e("BAD LOG: ", varfmt(log, color=True))
varprt(match, "find vlink")
return flows
def netusage(asc=True):
out = []
for r in g_routers.values():
for p in r.ports.values():
klog.d(varfmt(p, "NetUsage", True))
try:
d = DotDict()
ipaddr = p.get("__ipaddr__")
dbport = db.ports.find_one({"ip_str": ipaddr})
if not dbport:
klog.e("Port (%s) not found" % ipaddr)
if not conf.NETUSE_DEBUG:
continue
else:
d.port_uid = dbport.get("uid")
d.if_name = dbport.get("if_name")
d["__obj_db__"] = dbport
new = int(p.rti.new.ifHCOutOctets)
old = int(p.rti.old.ifHCOutOctets)
diff_bytes = new - old
diff_seconds = p.rti.new.time - p.rti.old.time
bw_in_bytes = int(p.ifHighSpeed) * 1000000 / 8
d.utilization = 100.0 * diff_bytes / diff_seconds / bw_in_bytes
d.__diff_seconds = diff_seconds
b = sc.tos(diff_bytes, "b", False, 3)
k = sc.tos(diff_bytes, "k", True, 3)
m = sc.tos(diff_bytes, "m", True, 3)
g = sc.tos(diff_bytes, "g", True, 3)
text = "%sB or %sK or %sM or %sG Bytes" % (b, k, m, g)
d.__diff_size = text
b = sc.tos(bw_in_bytes, "b", False, 3)
k = sc.tos(bw_in_bytes, "k", True, 3)
m = sc.tos(bw_in_bytes, "m", True, 3)
g = sc.tos(bw_in_bytes, "g", True, 3)
text = "%sB or %sK or %sM or %sG Bytes" % (b, k, m, g)
d.__bandwidth = text
d.ip = p.__ipaddr__
d.loopback = p.__loopback__
setp = g_setport_fr_ip.get(d.ip)
if setp:
d.port_uid = setp.uid
d.if_name = setp.if_name
out.append(d)
except:
continue
mul = 10000000000 if asc else -10000000000
return sorted(out, lambda x, y: int(mul * (x.utilization - y.utilization)))
def subcall(cls, cmd):
try:
return subprocess.check_output(cmd).replace("\r", "\n").split("\n")
except:
klog.e("CMD:%s\r\nBT:%s" % (cmd, traceback.format_exc()))
return []
def get_equip_flow(uid, limit=None, dir="o"):
flows = []
show = {
"_id": 0,
"bps": 1,
"bytes": 1,
"time.log": 1,
"time.last": 1,
"time.first": 1,
"addr.src": 1,
"addr.dst": 1,
"port.nexthop": 1,
"port.output": 1
}
equip = getequip(uid)
if not equip:
klog.e("No such equip, %s" % str(uid))
return {}
limit = limit or "40"
dir = dir or "io"
ip = equip["ip_str"]
orlis = []
if 'i' in dir:
orlis.append({"loopback.nxt": ip})
if 'o' in dir:
orlis.append({"loopback.cur": ip})
match = {"$or": orlis}
logs = db.logs.find(match).sort("bps",
pymongo.DESCENDING).limit(int(limit))
tmq = time.time()
for log in logs:
try:
timediff = time.time() - log["time"]["log"]
if timediff > conf.DB_LOG_AGE_SKIP:
continue
bps = log["bps"]
src = log["addr"]["src"]
dst = log["addr"]["dst"]
sportip = log["port"]["output"]
dportip = log["port"]["nexthop"]
src_equip = equip_fr_port_ip(sportip) or equip_fr_port_ip(
log["loopback"]["cur"])
dst_equip = equip_fr_port_ip(dportip) or equip_fr_port_ip(
log["loopback"]["nxt"])
seid = src_equip.get("uid", "<Null:output:%s>" % sportip)
seip = src_equip.get("ip_str", "<Null:output:%s>" % sportip)
spip = sportip
deid = dst_equip.get("uid", "<Null:nexthop:%s>" % dportip)
deip = dst_equip.get("ip_str", "<Null:nexthop:%s>" % dportip)
dpip = dportip
if ip == log["loopback"]["cur"]:
direct = "LOOP" if deid == uid else "OUT"
else:
direct = "IN"
# XXX: vlink = db.vlinks.find_one({"sportip": sportip, "dportip":
# dportip})
sloopbackip = src_equip.get("ip_str", "<Null>")
dloopbackip = dst_equip.get("ip_str", "<Null>")
vlink = db.vlinks.find_one({
"sloopbackip": sloopbackip,
"dloopbackip": dloopbackip
})
vlink_uid = vlink["uid"] if vlink else "<Bad>"
dirdic = {
"dir": direct,
"from": {
"equip_uid": seid,
"equip_ip": seip,
"port_ip": spip
},
"to": {
"equip_uid": deid,
"equip_ip": deip,
"port_ip": dpip
}
}
last = log["time"]["last"]
first = log["time"]["first"]
_bps = "{:.3f}G or {:.3f}M or {:.3f}K".format(
btog(bps), btom(bps), btok(bps))
_flow = "(%s) >>> (%s)@(%s) >>> (%s)@(%s) >>> (%s)" % (
src, spip, seip, dpip, deip, dst)
flow = {
"_flow": _flow,
"_bps": _bps,
"_dir": dirdic,
"_bytes": log["bytes"],
"bps": bps,
"src": src,
"dst": dst,
"next_hop_uid": deid,
"uid": vlink_uid,
"_time": "%d - %d = %d" % (last, first, last - first)
}
flows.append(flow)
except:
klog.e(traceback.format_exc())
klog.e("BAD LOG: ", varfmt(log, color=True))
tmh = time.time()
print "cost ...:", (tmh - tmq)
return flows
def rec2dic(routerip, rec):
try:
src_addr = "%d.%d.%d.%d" % (rec.src_addr_a, rec.src_addr_b,
rec.src_addr_c, rec.src_addr_d)
nexthop = "%d.%d.%d.%d" % (rec.nexthop_a, rec.nexthop_b,
rec.nexthop_c, rec.nexthop_d)
dst_addr = "%d.%d.%d.%d" % (rec.dst_addr_a, rec.dst_addr_b,
rec.dst_addr_c, rec.dst_addr_d)
in_if = rec.in_if
out_if = rec.out_if
packets = rec.packets
octets = rec.octets
first = rec.first
last = rec.last
src_port = rec.src_port
dst_port = rec.dst_port
# tcp_flags = rec.tcp_flags
# ip_proto = rec.ip_proto
# tos = rec.tos
# src_as = rec.src_as
# dst_as = rec.dst_as
# src_mask = rec.src_mask
# dst_mask = rec.dst_mask
# XXX: Skip if nextHop is 0.0.0.0
if nexthop == "0.0.0.0":
klog.w("Skip when nextHop is 0.0.0.0: (%s)" % str(rec))
return None, "nextHop is 0.0.0.0"
# XXX: Inf is the port send netflow package
# routerip is not must the loopback address
inf = spi.scget("", "ipaddr", routerip)
if not inf:
klog.e("ERR: Not found:", routerip)
return None, "Loopback not exists"
loopback = inf["__loopback__"]
dic = DotDict()
dic.loopback.cur = loopback
#
# Flow Info
#
dic.addr.src = src_addr
dic.addr.dst = dst_addr
dic.port.nexthop = nexthop
tmp = spi.scget("", "ipaddr", nexthop)
if tmp:
dic.loopback.nxt = tmp["__loopback__"]
else:
dic.loopback.nxt = "<%s: NotFound>" % (nexthop)
klog.e("NotFound: nexthop: ", nexthop)
return None, "nexthop not exists"
dic["bytes"] = octets
tmp = spi.scget(loopback, "ifindex", out_if)
if tmp:
dic.port.output = tmp["__ipaddr__"]
else:
dic.port.output = "<%s@%s>" % (out_if, loopback)
klog.e("NotFound: %s@%s" % (out_if, loopback))
tmp = spi.scget(loopback, "ifindex", in_if)
if tmp:
dic.port.input = tmp["__ipaddr__"]
else:
dic.port.input = "<%s@%s>" % (in_if, loopback)
klog.e("NotFound: %s@%s" % (in_if, loopback))
dic["_id"] = "{loopback.cur}::{addr.src}_to_{addr.dst}".format(
**dic)
diff = last - first
bps = 8.0 * int(dic["bytes"]) * conf.SAMPLE_RATE / diff * 1000
dic["bps"] = int(bps)
dic.time.last = last
dic.time.first = first
dic.time.log = time.time()
return dic, None
except:
klog.e(traceback.format_exc())
klog.e("Except rec:", varfmt(rec, color=True))
return None, "Except: %s" % traceback.format_exc()
def load_static(self, host, comm, vern):
'''Collector information according to router's information'''
downports = []
lines = sop.walk(host, comm, vern, oid_ipAdEntIfIndex)
for line in lines:
port_ipaddr, _, port_index = sop.splitline(line,
oid_ipAdEntIfIndex)
if not port_ipaddr:
continue
oid_ifAdminStatus = oid_ifAdminStatusBase + str(port_index)
_, _, state = sop.get(host, comm, vern, oid_ifAdminStatus)
if state != 1:
downports.append(port_ipaddr)
continue
#
# Get PortInfo
#
# XXX: ifindex number maybe same across different routers
hashkey = "%s#%s" % (host, port_index)
inf = self.portInfos.get(hashkey)
if not inf:
inf = DotDict()
self.portInfos[hashkey] = inf
klog.d("New portInfo: host:%s, ipaddr:%s" %
(host, port_ipaddr))
else:
klog.d("Found portInfo: host:%s, ipaddr:%s" %
(host, port_ipaddr))
inf["__loopback__"] = host
inf["__snmp_comm__"] = comm
inf["__snmp_vern__"] = vern
inf["__ipaddr__"] = port_ipaddr
for oid_base, name in self.oid_static.items():
oid = oid_base + "." + str(port_index)
_, _, value = sop.get(host, comm, vern, oid)
inf[name] = value
#
# Shortcuts
#
#
self.save_ipaddr(port_ipaddr, inf)
self.save_ifindex(host, inf["ifIndex"], inf)
# Save to db
db.devs.replace_one({"_id": hashkey}, dict(inf), True)
#
# Mark that this router has collected the static information
#
hashkeys = self.readyRouters.get(host, set())
hashkeys.add(hashkey)
self.readyRouters[host] = hashkeys
if downports:
klog.e("(%s) DownPorts: %s" % (host, downports))