def _find_limit(entries):
traffic_list = [
nfsenutils.traff_to_val(nfsenutils.get_traffic(
entry[nfsenutils.BYTES])) for entry in entries
]
traffic_list = set(traffic_list)
traffic_list = list(traffic_list)
traffic_list.sort(reverse=True)
limit = traffic_list[-2]
first_appearance = 0
for index, entry in enumerate(entries):
#print(nfsenutils.traff_to_val(entry[nfsenutils.BPS]))
if nfsenutils.traff_to_val(entry[nfsenutils.BYTES]) == limit:
#print("Ocurrence found")
first_appearance = index
break
ignore_list = [
ipaddress.IPv4Address(entry[nfsenutils.IP])
for entry in entries[first_appearance:]
]
return limit, ignore_list
def get_nfsen_available_as(interface, router, timestamp, nfsen):
query = {"filter": "in if %s" % interface,
"stattype": 11,
"topN": 3}
query["srcselector[]"] = router
nfsen_output = nfsen.query(query, timestamp)
if DEBUG:
print(nfsen_output)
nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output)
as_traffic = {}
has_as_zero = False
for entry in nfsen_dump:
as_num = entry[nfsenutils.AS]
if as_num in BAD_AS_LIST:
has_as_zero = True
continue
bps = nfsenutils.traff_to_val(entry[nfsenutils.BPS])
calc_bps = calculate_bw(entry)
as_traffic[as_num] = {"nfsen_bw": bps,
"calc_bw": calc_bps}
#print(as_traffic)
#print("HAS ZERO ", has_as_zero)
return as_traffic, has_as_zero
def calculate_bw(entry): #bits = bytes_in_G * 1024 * 1024 * 1024 * 8 bytes_count = nfsenutils.traff_to_val(nfsenutils.get_traffic(entry[nfsenutils.BYTES])) bits = bytes_count * 8 bps = bits/(60*5) return bps
def as_0_fix(nfsen, query, timestamp, as_resource, bad_as_list):
as_num = as_resource["match"]
if DEBUG:
print("AS 0 FIX %s AS %s<----------------" % (as_resource["name"], as_num))
as_range_filter = AS_RANGE_FILTERS.get(as_num,[])
if not as_range_filter:
return 0
as_usage = 0
for bad_as in bad_as_list:
for range_filter in as_range_filter.get("filters",[]):
range_query = dict(query)
range_query["filter"] = f"({query['filter']}) and src as {bad_as} and ({range_filter})"
if len(range_query["filter"]) > 10240:
print("ERROR: FILTER TOO BIG. %s length." % len(range_query["filter"]))
raise
nfsen_output = nfsen.query(range_query, timestamp)
if DEBUG:
print(nfsen_output)
nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output)
evaluated_entry = search_nfsen_entry(nfsen_dump, bad_as)
if evaluated_entry is not None:
resource_raw = int(round(nfsenutils.traff_to_val(evaluated_entry[nfsenutils.BPS]), 0))
# TODO: It should probably be a good idea to check if is too low or too wild and adjust.
else:
resource_raw = 0
as_usage += resource_raw
print("AS 0 CUMULATED AS %s USAGE: %s------------>" % (bad_as, as_usage))
if DEBUG:
print("AS 0 USAGE: %s------------>" % as_usage)
return as_usage
def peru_analyzer(timestamp):
load_as_range_filters()
db = intialize_influx()
nfsen = nfsenutils.NFSENServer("https://nfsen.us.iptp.net")
nfsen.login()
# if earlier then 23 hours ago...
if timestamp < time.time() - 23 * 60 * 60:
nfsen.set_window_size(3) # 4 days. This is as back as nfsen alow us to go. November 2020
report = []
for country in COUNTRIES:
if DEBUG:
print("---------------COUNTRY: %s-------------" % country["name"])
country_data = {"name": country["name"], "customers": [], "extra_resources": []}
for customer in country["customers"]:
if DEBUG:
print("***************** CUSTOMER: %s ***********************" % customer["name"])
customer_data = {"name": customer["name"], "resource_usage": {}}
for resource_type, resource_info in customer["resources"].items() :
if DEBUG:
print("================= Reource: %s ***********************" % resource_type)
customer_data["resource_usage"][resource_type] = {}
query = dict(resource_info["query"])
query["srcselector[]"] = country["router"]
nfsen_output = nfsen.query(query, timestamp)
if DEBUG:
print(nfsen_output)
nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output)
bad_as_list = []
if "as" in resource_type:
for bad_as in BAD_AS_LIST:
as_entry = search_nfsen_entry(nfsen_dump, bad_as)
if as_entry:
bad_as_list.append(bad_as)
if DEBUG:
print(country["resources"][resource_type])
for resource in country["resources"][resource_type]:
evaluated_entry = search_nfsen_entry(nfsen_dump, resource["match"])
if evaluated_entry is not None:
resource_raw = int(round(nfsenutils.traff_to_val(evaluated_entry[nfsenutils.BPS]),0))
else:
resource_raw = 0
if "as" in resource_type and bad_as_list:
as_0_usage = as_0_fix(nfsen, query, timestamp, resource, bad_as_list)
resource_raw += as_0_usage
if DEBUG:
print(customer_data)
customer_data["resource_usage"][resource_type][resource["name"]] = resource_raw
country_data["customers"].append(customer_data)
to_influx(db, int(timestamp), customer_data, country["name"])
for extra_resource in country.get("extra_resources", []):
if DEBUG:
print("***************** EXTRA RESOURCE: %s ***********************" % extra_resource["name"])
#customer_data = {"name": extra_resource["name"], "resource_usage": {}}
resource_type = extra_resource["resource_type"]
#customer_data["resource_usage"][resource_type] = {}
query = dict(extra_resource["query"])
query["srcselector[]"] = country["router"]
nfsen_output = nfsen.query(query, timestamp)
if DEBUG:
print(nfsen_output)
nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output)
for customer in country["customers"]:
evaluated_entry = search_nfsen_entry(nfsen_dump, customer["interface"])
if evaluated_entry is not None:
resource_raw = int(round(nfsenutils.traff_to_val(evaluated_entry[nfsenutils.BPS]), 0))
else:
resource_raw = 0
#customer_data["resource_usage"][resource_type][customer["name"]] = resource_raw
customer_data = {"name": customer["name"],
"resource_usage": {extra_resource["resource_type"]: {extra_resource["name"]: resource_raw}}}
if DEBUG:
print(customer_data)
to_influx(db, int(timestamp), customer_data, country["name"])
country_data["extra_resources"].append(customer_data)
report.append(country_data)
if DEBUG:
pprint(report)
def as_explorer(interface, router, AS_CAPTURE=None, timestamp=None, silent=False):
if timestamp is None:
timestamp = int(time.time()) - (5*60)
nfsen = nfsenutils.NFSENServer("https://nfsen.us.iptp.net")
nfsen.login()
as_traffic, has_as_zero = get_nfsen_available_as(interface, router, timestamp, nfsen)
if has_as_zero:
#limit_size is the parameter to change in bps
query = {"filter": "in if %s and (src as 0 or src as 3549)" % interface,
"stattype": 2,
"srcselector[]": router,
"topN": 5, # Top 500
#"statorder": 4 # Order by bps
#"limitoutput": "checked" # Limit True
#"limitwhat": 1, # Limit according to Traffic
#"limitsize": 0 # Limit to at max 0 bps
}
#{traffic: "ETC NETWORKS"}
#as_traffic = {}
as_captured = []
ignore_list = []
unknown = []
unknown_nfsen_traff = 0
unknown_calc_traff = 0
passes = 0
#while True:
for a in range(3):
passes += 1
nfsen_output = nfsen.query(query, timestamp)
if DEBUG:
print(nfsen_output)
nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output)
whoip.whoipe(nfsen_dump)
networks = load_cache()
for entry in nfsen_dump:
ip = ipaddress.IPv4Address(entry[nfsenutils.IP])
if ip in ignore_list:
continue
bps = nfsenutils.traff_to_val(entry[nfsenutils.BPS])
calc_bps = calculate_bw(entry)
as_num = None
for net in networks:
if ip in net["network"]:
as_num = net["as_num"]
break
if as_num is None:
unknown.append(ip)
unknown_nfsen_traff += bps
unknown_calc_traff += calc_bps
else:
if not as_traffic.get(as_num, 0):
as_traffic[as_num] = {"nfsen_bw": 0,
"calc_bw": 0}
as_traffic[as_num] = {"nfsen_bw": as_traffic[as_num]["nfsen_bw"] + bps,
"calc_bw": as_traffic[as_num]["calc_bw"] + calc_bps}
if as_num == AS_CAPTURE:
as_captured.append(ip)
if len(nfsen_dump) < 500:
if DEBUG:
print("NO MORE DUMPS, FINISHING")
break
limit, ignore_list = _find_limit(nfsen_dump)
print(limit)
#pprint(ignore_list)
query["limitoutput"] = "checked" # Limit True
query["limitwhat"] = 1 # Limit according to Traffic
query["limithow"] = 1 # Limit according to Traffic
query["limitsize"] = int(round(limit, 0)) # Limit to at max bps
as_traffic = {k: v for k, v in sorted(as_traffic.items(),
key=lambda item: item[1]["calc_bw"])}
#key=lambda item: item[1]["nfsen_bw"])}
load_as_cache()
#pprint(as_traffic)
result_traffic = copy.deepcopy(as_traffic)
for as_num, traffic in as_traffic.items():
#print(traffic)
ratio = round((traffic["nfsen_bw"] / traffic["calc_bw"]) * 100, 2)
nfsen_traffic = int(round(traffic["nfsen_bw"], 0))
as_traffic[as_num]["nfsen_bw"] = nfsenutils.val_to_traff(nfsen_traffic)
calc_traffic = int(round(traffic["calc_bw"], 0))
as_traffic[as_num]["calc_bw"] = nfsenutils.val_to_traff(calc_traffic)
as_name = get_AS_name(as_num)
result_traffic[as_num]["nfsen_bw"] = int(round(result_traffic[as_num]["nfsen_bw"],0))
result_traffic[as_num]["calc_bw"] = int(round(result_traffic[as_num]["calc_bw"],0))
result_traffic[as_num]["ratio"] = ratio
result_traffic[as_num]["as_name"] = as_name
if not silent:
print("{:<8} {:<30} {:>10} {:>10} {:>10}%".format(as_num, as_name,
as_traffic[as_num]["nfsen_bw"],
as_traffic[as_num]["calc_bw"],
ratio))
save_as_cache()
if silent:
return result_traffic
#print("///////////////////////////////////////////////////")
#pprint(as_traffic)
unknown_nfsen_traff = int(round(unknown_nfsen_traff, 0))
unknown_nfsen_traff = nfsenutils.val_to_traff(unknown_nfsen_traff)
unknown_calc_traff = int(round(unknown_calc_traff, 0))
unknown_calc_traff = nfsenutils.val_to_traff(unknown_calc_traff)
print("TOTAL PASSES: %s" % passes)
pprint("Unknown traffic: %s" % unknown_nfsen_traff)
pprint("Unknown CALC traffic: %s" % unknown_calc_traff)
pprint("Unknowns: " )
pprint(unknown)
print("AS Captured: ")
pprint(as_captured)
return result_traffic