def flowv(cmdctx, calldic):
'''Show flow for vlink
.opt --l limit :100
.opt --hi bpsMin :9999999999999
.opt --lo bpsMax :0
.opt --s searchPat :.*
.arg uids
uid list
'''
limit = int(calldic.nth_opt("l", 0, 100))
bpshi = int(calldic.nth_opt("hi", 0, 9999999999999))
bpslo = int(calldic.nth_opt("lo", 0, 0))
uids = calldic.get_args()
if not uids:
uids = [e.get("uid") for e in db.vlinks.find()]
pat = calldic.nth_opt("s", 0, ".*")
pat = re.compile(pat)
klog.d(varfmt(uids, "UID List"))
res = []
for uid in uids:
res.extend(get_vlink_flow(uid))
res = [f for f in res if pat.search(str(f))]
res = sorted(res, lambda x, y: y.get("bps", 0) - x.get("bps", 0))
res = filter(lambda x: bpslo <= x.get("bps", 0) <= bpshi, res)
return res[:limit]
def _on_recive(self, data, address):
ver, = unpack(">H", data[:2])
klog.d("Found a package: Version: ", ver)
if ver == 5:
return self._on_pkg_v5(data, address)
if ver == 9:
return self._on_pkg_v9(data, address)
def load(self):
if not self.ports:
klog.d("load static for: %s" % self.host)
self.load_static()
klog.d("load runtime for: %s" % self.host)
for p in self.ports.values():
self.load_runtime(p)
def load(self, host, comm, vern="2c"):
if host not in self.readyRouters:
klog.d("load static for: %s" % host)
self.load_static(host, comm, vern)
klog.d("load runtime for: %s" % host)
hashkeys = self.readyRouters.get(host, set()).copy()
for hashkey in hashkeys:
inf = self.portInfos.get(hashkey)
self.load_port_info(inf)
def get_vlink_flow(uid):
flows = []
show = {"_id": 0, "bps": 1, "time.log": 1, "addr.src": 1, "addr.dst": 1}
# vlink.sport, vlink.dport => db.ports.ui
vlink = getvlink(uid)
if not vlink:
klog.e("NotFound: vlink: uid:", uid)
return []
klog.d(varfmt(vlink, "vlink"))
sportip = vlink.get("sportip")
dportip = vlink.get("dportip")
src_equip = equip_fr_port_ip(sportip) or equip_fr_port_ip(
vlink["sloopbackip"])
dst_equip = equip_fr_port_ip(dportip) or equip_fr_port_ip(
vlink["dloopbackip"])
if not src_equip:
klog.e("NotFound: src_equip: sportip:", sportip)
return []
next_hop_uid = dst_equip.get("uid", "<Null>")
match = {"loopback.cur": src_equip["ip_str"], "port.nexthop": dportip}
klog.d(varfmt(match, "db.logs.find: match"))
logs = db.logs.find(match, show)
for log in logs:
try:
timediff = time.time() - log["time"]["log"]
if timediff > conf.DB_LOG_AGE_SKIP:
continue
_flow = "(%s) >>> (%s)@(%s) >>> (%s)@(%s) >>> (%s)" % (
log["addr"]["src"], sportip, src_equip["ip_str"], dportip,
dst_equip["ip_str"], log["addr"]["dst"])
flow = {
"_flow": _flow,
"bps": log["bps"],
"src": log["addr"]["src"],
"dst": log["addr"]["dst"],
"uid": uid,
"next_hop_uid": next_hop_uid
}
flows.append(flow)
except:
klog.e(traceback.format_exc())
klog.e("BAD LOG: ", varfmt(log, color=True))
varprt(match, "find vlink")
return flows
def act(self):
lc_loops.inc()
ctime = time.time() - self.keeptime
query = {"time.log": {"$lt": ctime}}
res = db.logs.delete_many(query)
klog.d("%s" % varfmt(query))
klog.d("%d deleted." % res.deleted_count)
lc_logs.inc(res.deleted_count)
return self.keeptime * 9 / 10
def docmd_flow():
calldic = idic()
klog.d(varfmt(calldic, "calldic"))
request = calldic["request"]
if request == "ms_flow_set_topo":
return ms_flow_set_topo(calldic)
if request == "ms_flow_get_flow":
return ms_flow_get_flow(calldic)
return "Bad request '%s'" % request
def unload(self, host):
hashkeys = self.readyRouters.get(host)
if not hashkeys:
klog.e("unload router %s failed, it not loaded yet" % host)
return
klog.d("unload router: %s" % host)
for hashkey in hashkeys:
try:
del self.portInfos[hashkey]
except:
pass
try:
del self.readyRouters[host]
except:
pass
def _on_pkg_v5(self, data, address):
##
## Header
##
## u_int16_t version;
## u_int16_t count;
## u_int32_t sysup_time;
## u_int32_t unix_secs;
## u_int32_t unix_nsecs;
## u_int32_t flow_sequence;
## u_int8_t engine_type;
## u_int8_t engine_id;
## u_int16_t sampling;
fmt = '>HHIIIIBBH'
hdr = HdrV5(*unpack(fmt, data[:24]))
klog.d(varfmt(hdr, color=True))
##
## Record
##
ofs = 24
reclen = 48
for i in range(hdr.count):
## struct in_addr src_addr;
## struct in_addr dst_addr;
## struct in_addr nexthop;
## u_int16_t in_if;
## u_int16_t out_if;
## u_int32_t packets;
## u_int32_t octets;
## u_int32_t first;
## u_int32_t last;
## u_int16_t src_port;
## u_int16_t dst_port;
## u_int8_t pad1;
## u_int8_t tcp_flags;
## u_int8_t ip_proto;
## u_int8_t tos;
## u_int16_t src_as;
## u_int16_t dst_as;
## u_int8_t src_mask;
## u_int8_t dst_mask;
## u_int16_t pad2;
fmt = '>BBBBBBBBBBBBHHIIIIHHBBBBHHBBH'
rec = RecV5(*unpack(fmt, data[ofs:ofs + reclen]))
klog.d(varfmt(rec, color=True))
ofs += reclen
if self.onflow:
segs = []
segs.append(address[0])
segs.append(time.strftime("%Y-%m-%d %H:%M:%S"))
segs.append("%d.%d.%d.%d" % (rec.src_addr_a, rec.src_addr_b,
rec.src_addr_c, rec.src_addr_d))
segs.append("%d.%d.%d.%d" % (rec.dst_addr_a, rec.dst_addr_b,
rec.dst_addr_c, rec.dst_addr_d))
segs.append("%d.%d.%d.%d" % (rec.nexthop_a, rec.nexthop_b,
rec.nexthop_c, rec.nexthop_d))
segs.append(rec.packets)
segs.append(rec.octets)
segs.append(rec.src_mask)
segs.append(rec.dst_mask)
segs.append(rec.out_if)
segs.append(rec.in_if)
segs.append(rec.src_port)
segs.append(rec.dst_port)
segs.append("5")
log = ""
log += "NEWLOG: %s: " % address[0]
log += "%d.%d.%d.%d " % (rec.src_addr_a, rec.src_addr_b,
rec.src_addr_c, rec.src_addr_d)
log += ">> "
log += "%d.%d.%d.%d " % (rec.nexthop_a, rec.nexthop_b,
rec.nexthop_c, rec.nexthop_d)
log += ">> "
log += "%d.%d.%d.%d " % (rec.dst_addr_a, rec.dst_addr_b,
rec.dst_addr_c, rec.dst_addr_d)
log += "*********"
klog.d(log)
self.onflow(address[0], rec)
def netusage(asc=True):
out = []
for r in g_routers.values():
for p in r.ports.values():
klog.d(varfmt(p, "NetUsage", True))
try:
d = DotDict()
ipaddr = p.get("__ipaddr__")
dbport = db.ports.find_one({"ip_str": ipaddr})
if not dbport:
klog.e("Port (%s) not found" % ipaddr)
if not conf.NETUSE_DEBUG:
continue
else:
d.port_uid = dbport.get("uid")
d.if_name = dbport.get("if_name")
d["__obj_db__"] = dbport
new = int(p.rti.new.ifHCOutOctets)
old = int(p.rti.old.ifHCOutOctets)
diff_bytes = new - old
diff_seconds = p.rti.new.time - p.rti.old.time
bw_in_bytes = int(p.ifHighSpeed) * 1000000 / 8
d.utilization = 100.0 * diff_bytes / diff_seconds / bw_in_bytes
d.__diff_seconds = diff_seconds
b = sc.tos(diff_bytes, "b", False, 3)
k = sc.tos(diff_bytes, "k", True, 3)
m = sc.tos(diff_bytes, "m", True, 3)
g = sc.tos(diff_bytes, "g", True, 3)
text = "%sB or %sK or %sM or %sG Bytes" % (b, k, m, g)
d.__diff_size = text
b = sc.tos(bw_in_bytes, "b", False, 3)
k = sc.tos(bw_in_bytes, "k", True, 3)
m = sc.tos(bw_in_bytes, "m", True, 3)
g = sc.tos(bw_in_bytes, "g", True, 3)
text = "%sB or %sK or %sM or %sG Bytes" % (b, k, m, g)
d.__bandwidth = text
d.ip = p.__ipaddr__
d.loopback = p.__loopback__
setp = g_setport_fr_ip.get(d.ip)
if setp:
d.port_uid = setp.uid
d.if_name = setp.if_name
out.append(d)
except:
continue
mul = 10000000000 if asc else -10000000000
return sorted(out, lambda x, y: int(mul * (x.utilization - y.utilization)))
def docmd_ms_link_links():
calldic = idic()
klog.d(varfmt(calldic, "calldic"))
return g_cmdmap.get(calldic.request, "default")(calldic)
def act(self):
'''Fetch and save to db'''
klog.d("Collector, acting...")
self.update()
return 10
def __init__(self, name="SnmpCollector"):
klog.d("INTO Collector")
miethread.MieThread.__init__(self, name=name)
self.start()
def load_each(self, r):
klog.d("Loading ...:", r.host)
r.load()
def update(self):
'''Scan mango and generate ifindex number and port ipaddr'''
for dic in db.equips.find({}, {"_id": 0, "ip_str": 1, "community": 1}):
klog.d(varfmt(dic))
DeferDo(self.load_each, dic)
def load_static(self, host, comm, vern):
'''Collector information according to router's information'''
downports = []
lines = sop.walk(host, comm, vern, oid_ipAdEntIfIndex)
for line in lines:
port_ipaddr, _, port_index = sop.splitline(line,
oid_ipAdEntIfIndex)
if not port_ipaddr:
continue
oid_ifAdminStatus = oid_ifAdminStatusBase + str(port_index)
_, _, state = sop.get(host, comm, vern, oid_ifAdminStatus)
if state != 1:
downports.append(port_ipaddr)
continue
#
# Get PortInfo
#
# XXX: ifindex number maybe same across different routers
hashkey = "%s#%s" % (host, port_index)
inf = self.portInfos.get(hashkey)
if not inf:
inf = DotDict()
self.portInfos[hashkey] = inf
klog.d("New portInfo: host:%s, ipaddr:%s" %
(host, port_ipaddr))
else:
klog.d("Found portInfo: host:%s, ipaddr:%s" %
(host, port_ipaddr))
inf["__loopback__"] = host
inf["__snmp_comm__"] = comm
inf["__snmp_vern__"] = vern
inf["__ipaddr__"] = port_ipaddr
for oid_base, name in self.oid_static.items():
oid = oid_base + "." + str(port_index)
_, _, value = sop.get(host, comm, vern, oid)
inf[name] = value
#
# Shortcuts
#
#
self.save_ipaddr(port_ipaddr, inf)
self.save_ifindex(host, inf["ifIndex"], inf)
# Save to db
db.devs.replace_one({"_id": hashkey}, dict(inf), True)
#
# Mark that this router has collected the static information
#
hashkeys = self.readyRouters.get(host, set())
hashkeys.add(hashkey)
self.readyRouters[host] = hashkeys
if downports:
klog.e("(%s) DownPorts: %s" % (host, downports))
def savedic(dic):
klog.d("Saving: ", varfmt(dic, color=True))
db.logs.replace_one({"_id": dic["_id"]}, dic.todic(), True)
