def get_nfsen_available_as(interface, router, timestamp, nfsen): query = {"filter": "in if %s" % interface, "stattype": 11, "topN": 3} query["srcselector[]"] = router nfsen_output = nfsen.query(query, timestamp) if DEBUG: print(nfsen_output) nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output) as_traffic = {} has_as_zero = False for entry in nfsen_dump: as_num = entry[nfsenutils.AS] if as_num in BAD_AS_LIST: has_as_zero = True continue bps = nfsenutils.traff_to_val(entry[nfsenutils.BPS]) calc_bps = calculate_bw(entry) as_traffic[as_num] = {"nfsen_bw": bps, "calc_bw": calc_bps} #print(as_traffic) #print("HAS ZERO ", has_as_zero) return as_traffic, has_as_zero
def as_0_fix(nfsen, query, timestamp, as_resource, bad_as_list): as_num = as_resource["match"] if DEBUG: print("AS 0 FIX %s AS %s<----------------" % (as_resource["name"], as_num)) as_range_filter = AS_RANGE_FILTERS.get(as_num,[]) if not as_range_filter: return 0 as_usage = 0 for bad_as in bad_as_list: for range_filter in as_range_filter.get("filters",[]): range_query = dict(query) range_query["filter"] = f"({query['filter']}) and src as {bad_as} and ({range_filter})" if len(range_query["filter"]) > 10240: print("ERROR: FILTER TOO BIG. %s length." % len(range_query["filter"])) raise nfsen_output = nfsen.query(range_query, timestamp) if DEBUG: print(nfsen_output) nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output) evaluated_entry = search_nfsen_entry(nfsen_dump, bad_as) if evaluated_entry is not None: resource_raw = int(round(nfsenutils.traff_to_val(evaluated_entry[nfsenutils.BPS]), 0)) # TODO: It should probably be a good idea to check if is too low or too wild and adjust. else: resource_raw = 0 as_usage += resource_raw print("AS 0 CUMULATED AS %s USAGE: %s------------>" % (bad_as, as_usage)) if DEBUG: print("AS 0 USAGE: %s------------>" % as_usage) return as_usage
def peru_analyzer(timestamp): load_as_range_filters() db = intialize_influx() nfsen = nfsenutils.NFSENServer("https://nfsen.us.iptp.net") nfsen.login() # if earlier then 23 hours ago... if timestamp < time.time() - 23 * 60 * 60: nfsen.set_window_size(3) # 4 days. This is as back as nfsen alow us to go. November 2020 report = [] for country in COUNTRIES: if DEBUG: print("---------------COUNTRY: %s-------------" % country["name"]) country_data = {"name": country["name"], "customers": [], "extra_resources": []} for customer in country["customers"]: if DEBUG: print("***************** CUSTOMER: %s ***********************" % customer["name"]) customer_data = {"name": customer["name"], "resource_usage": {}} for resource_type, resource_info in customer["resources"].items() : if DEBUG: print("================= Reource: %s ***********************" % resource_type) customer_data["resource_usage"][resource_type] = {} query = dict(resource_info["query"]) query["srcselector[]"] = country["router"] nfsen_output = nfsen.query(query, timestamp) if DEBUG: print(nfsen_output) nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output) bad_as_list = [] if "as" in resource_type: for bad_as in BAD_AS_LIST: as_entry = search_nfsen_entry(nfsen_dump, bad_as) if as_entry: bad_as_list.append(bad_as) if DEBUG: print(country["resources"][resource_type]) for resource in country["resources"][resource_type]: evaluated_entry = search_nfsen_entry(nfsen_dump, resource["match"]) if evaluated_entry is not None: resource_raw = int(round(nfsenutils.traff_to_val(evaluated_entry[nfsenutils.BPS]),0)) else: resource_raw = 0 if "as" in resource_type and bad_as_list: as_0_usage = as_0_fix(nfsen, query, timestamp, resource, bad_as_list) resource_raw += as_0_usage if DEBUG: print(customer_data) customer_data["resource_usage"][resource_type][resource["name"]] = resource_raw country_data["customers"].append(customer_data) to_influx(db, int(timestamp), customer_data, country["name"]) for extra_resource in country.get("extra_resources", []): if DEBUG: print("***************** EXTRA RESOURCE: %s ***********************" % extra_resource["name"]) #customer_data = {"name": extra_resource["name"], "resource_usage": {}} resource_type = extra_resource["resource_type"] #customer_data["resource_usage"][resource_type] = {} query = dict(extra_resource["query"]) query["srcselector[]"] = country["router"] nfsen_output = nfsen.query(query, timestamp) if DEBUG: print(nfsen_output) nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output) for customer in country["customers"]: evaluated_entry = search_nfsen_entry(nfsen_dump, customer["interface"]) if evaluated_entry is not None: resource_raw = int(round(nfsenutils.traff_to_val(evaluated_entry[nfsenutils.BPS]), 0)) else: resource_raw = 0 #customer_data["resource_usage"][resource_type][customer["name"]] = resource_raw customer_data = {"name": customer["name"], "resource_usage": {extra_resource["resource_type"]: {extra_resource["name"]: resource_raw}}} if DEBUG: print(customer_data) to_influx(db, int(timestamp), customer_data, country["name"]) country_data["extra_resources"].append(customer_data) report.append(country_data) if DEBUG: pprint(report)
def as_explorer(interface, router, AS_CAPTURE=None, timestamp=None, silent=False): if timestamp is None: timestamp = int(time.time()) - (5*60) nfsen = nfsenutils.NFSENServer("https://nfsen.us.iptp.net") nfsen.login() as_traffic, has_as_zero = get_nfsen_available_as(interface, router, timestamp, nfsen) if has_as_zero: #limit_size is the parameter to change in bps query = {"filter": "in if %s and (src as 0 or src as 3549)" % interface, "stattype": 2, "srcselector[]": router, "topN": 5, # Top 500 #"statorder": 4 # Order by bps #"limitoutput": "checked" # Limit True #"limitwhat": 1, # Limit according to Traffic #"limitsize": 0 # Limit to at max 0 bps } #{traffic: "ETC NETWORKS"} #as_traffic = {} as_captured = [] ignore_list = [] unknown = [] unknown_nfsen_traff = 0 unknown_calc_traff = 0 passes = 0 #while True: for a in range(3): passes += 1 nfsen_output = nfsen.query(query, timestamp) if DEBUG: print(nfsen_output) nfsen_dump = nfsenutils.get_sdump_entries(nfsen_output) whoip.whoipe(nfsen_dump) networks = load_cache() for entry in nfsen_dump: ip = ipaddress.IPv4Address(entry[nfsenutils.IP]) if ip in ignore_list: continue bps = nfsenutils.traff_to_val(entry[nfsenutils.BPS]) calc_bps = calculate_bw(entry) as_num = None for net in networks: if ip in net["network"]: as_num = net["as_num"] break if as_num is None: unknown.append(ip) unknown_nfsen_traff += bps unknown_calc_traff += calc_bps else: if not as_traffic.get(as_num, 0): as_traffic[as_num] = {"nfsen_bw": 0, "calc_bw": 0} as_traffic[as_num] = {"nfsen_bw": as_traffic[as_num]["nfsen_bw"] + bps, "calc_bw": as_traffic[as_num]["calc_bw"] + calc_bps} if as_num == AS_CAPTURE: as_captured.append(ip) if len(nfsen_dump) < 500: if DEBUG: print("NO MORE DUMPS, FINISHING") break limit, ignore_list = _find_limit(nfsen_dump) print(limit) #pprint(ignore_list) query["limitoutput"] = "checked" # Limit True query["limitwhat"] = 1 # Limit according to Traffic query["limithow"] = 1 # Limit according to Traffic query["limitsize"] = int(round(limit, 0)) # Limit to at max bps as_traffic = {k: v for k, v in sorted(as_traffic.items(), key=lambda item: item[1]["calc_bw"])} #key=lambda item: item[1]["nfsen_bw"])} load_as_cache() #pprint(as_traffic) result_traffic = copy.deepcopy(as_traffic) for as_num, traffic in as_traffic.items(): #print(traffic) ratio = round((traffic["nfsen_bw"] / traffic["calc_bw"]) * 100, 2) nfsen_traffic = int(round(traffic["nfsen_bw"], 0)) as_traffic[as_num]["nfsen_bw"] = nfsenutils.val_to_traff(nfsen_traffic) calc_traffic = int(round(traffic["calc_bw"], 0)) as_traffic[as_num]["calc_bw"] = nfsenutils.val_to_traff(calc_traffic) as_name = get_AS_name(as_num) result_traffic[as_num]["nfsen_bw"] = int(round(result_traffic[as_num]["nfsen_bw"],0)) result_traffic[as_num]["calc_bw"] = int(round(result_traffic[as_num]["calc_bw"],0)) result_traffic[as_num]["ratio"] = ratio result_traffic[as_num]["as_name"] = as_name if not silent: print("{:<8} {:<30} {:>10} {:>10} {:>10}%".format(as_num, as_name, as_traffic[as_num]["nfsen_bw"], as_traffic[as_num]["calc_bw"], ratio)) save_as_cache() if silent: return result_traffic #print("///////////////////////////////////////////////////") #pprint(as_traffic) unknown_nfsen_traff = int(round(unknown_nfsen_traff, 0)) unknown_nfsen_traff = nfsenutils.val_to_traff(unknown_nfsen_traff) unknown_calc_traff = int(round(unknown_calc_traff, 0)) unknown_calc_traff = nfsenutils.val_to_traff(unknown_calc_traff) print("TOTAL PASSES: %s" % passes) pprint("Unknown traffic: %s" % unknown_nfsen_traff) pprint("Unknown CALC traffic: %s" % unknown_calc_traff) pprint("Unknowns: " ) pprint(unknown) print("AS Captured: ") pprint(as_captured) return result_traffic