def check_names(self, rule, expression, current, i, encounter_type, str_to_chk, ixp_long, ixp_short): ''' Checks the similarity of the concatenated numbers in case of IXP_IP keyword for consecutive IXP IPs in the path. It also compares the IXP short and long names. Input: a) rule: The current rule. b) expression: The current condition of the condition part of the rule. c) i: The hop of the current condition in the rule. d) encounter_type: The resolved IP path based on the encountered types (IXP IP, IXP Prefix, Normal IP, Illegal IP). e) str_to_chk: The condition keyword of the rule. d) ixp_long,ixp_short: The IXP long and short names. Output: True if the condition is satisfied, False otherwise. ''' string_handle = string_handler.string_handler() if len(rule) > i + 1 and len(ixp_long) > current + 1: [final1, final2] = self.find_numbers( rule, str_to_chk, current, True) if final1 == '' or final2 == '': return True if self.is_int(final1) and self.is_int(final2): flag = (string_handle.string_comparison(ixp_long[current], ixp_long[ current + 1]) or string_handle.string_comparison(ixp_short[current], ixp_short[current + 1])) if (final1 == final2 and not flag) or (final1 != final2 and flag): return False return True
def check_edges(self, rule, path_asn, current, str_to_chk, ixp_long, ixp_short): ''' Checks the similarity of the concatenated numbers in case of AS_M and IXP_IP keywords of the border hops of a rule with hop window of size three. Input: a) rule: The current rule. b) path_asn: The AS path. c) current: The current hop in the path. d) str_to_chk: The condition keyword of the rule. e) ixp_long,ixp_short: The long and short IXP names. Output: True if the condition is satisfied, False otherwise. ''' [final1, final2] = self.find_numbers(rule, str_to_chk, current, False) if final1 == '' or final2 == '': return True if self.is_int(final1) and self.is_int(final2) and 'AS_M' in str_chk: if (final1 == final2 and path_asn[current - 1] != path_asn[current + 1]) or (final1 != final2 and path_asn[current - 1] == path_asn[current + 1]): return False elif self.is_int(final1) and self.is_int(final2) and 'IXP_IP' in str_chk: string_hanlde = string_handler.string_handler() flag = (string_handle.string_comparison(ixp_long[current - 1], ixp_long[ current + 1]) or string_handle.string_comparison(ixp_short[current - 1], ixp_short[current + 1])) if (final1 == final2 and not flag) or (final1 != final2 and flag): return False return True
def print_traIXroute_dest(self, traixparser, db_extract, dns_print, dst_ip, src_ip='', info=''): ''' Prints traIXroute destination. Input: a) dns_print: Flag to enable resolving IP or FQDN. b) dst_ip: The destination IP/FQDN to probe. c) src_ip: The IP that issued the probe (optional). d) info: Traceroute path description. ''' string_handle = string_handler.string_handler() print_data = 'traIXroute' dns_name = '*' output_IP = '*' if string_handle.is_valid_ip_address(dst_ip, 'IP', 'CLI'): if dns_print: try: dns_name = socket.gethostbyaddr(dst_ip)[0] except: pass output_IP = dst_ip else: if dns_print: try: output_IP = socket.gethostbyname(dst_ip) except: pass dns_name = dst_ip if src_ip != '': origin_dns = '*' if dns_print: try: origin_dns = socket.gethostbyaddr(src_ip)[0] except: pass print_data += ' from ' + origin_dns + ' (' + src_ip + ')' if traixparser.flags['asn']: if src_ip in db_extract.asn_routeviews: print_data += ' AS' + db_extract.asn_routeviews[src_ip] elif src_ip: print_data += ' AS*' print_data += ' to ' + dns_name + ' (' + output_IP + ')' if traixparser.flags['asn']: if output_IP in db_extract.asn_routeviews: print_data += ' AS' + db_extract.asn_routeviews[output_IP] else: print_data += ' AS*' if info != '': print_data += ' info: ' + info self.measurement_info += print_data + '\n'
def include_additional(self, final_sub2name, subTree, additional_subnet2name, final_subnet2country, additional_pfx2cc, help_tree): ''' Includes the additional info subnet2names to the final dictionaries. Input: a) final_sub2name: The final {IXP Subnet}=[[IXP name long, IXP name short]] dictionary. b) subTree: The final [IXP Subnet]=[IXP name long, IXP name short]. c) additional_subnet2name: The user imported {IXP Subnet}=[[IXP name long, IXP name short]]. d) help_tree: SubneTree containing the IXP Subnets. e) final_subnet2country: subTree: The final [IXP Subnet]=[[IXP name long, IXP name short]]. Output: a) subTree: The final [IXP Subnet]=[[IXP name long, IXP name short]]. b) final_sub2name: The final dictionary with {IXP Subnet} = [[IXP long name, IXP short name]]. c) final_subnet2country: A dictioanry containing {IXP Subnet} = [ IXP country, IXP city]. ''' handle = string_handler.string_handler() for pfx in additional_subnet2name: while handle.sub_prefix_check(pfx, help_tree): subTree.remove(help_tree[pfx]) final_sub2name.pop(help_tree[pfx]) final_subnet2country.pop(help_tree[pfx]) help_tree.remove(help_tree[pfx]) final_sub2name[pfx] = [additional_subnet2name[pfx]] subTree[pfx] = [additional_subnet2name[pfx]] final_subnet2country[pfx] = additional_pfx2cc[pfx] return (subTree, final_sub2name, final_subnet2country)
def extract_ip(self, json_ip, temp_subnet_tree, reserved_tree): ''' Extracts the IXP IPs from peeringdb. Input: a) json_IP: A json table containing IXP IPs, IXP short names and IXP IDs. b) temp_subnet_Tree: The Subnet Tree containing the IXP subnets from peeringdb. c) reserved_tree: The SubnetTree containing the reserved Subnets. Output: a) ixp_to_asn: A dictionary with {IXP IP}=[ASN]. ''' handler = string_handler.string_handler() ixp_to_asn = {} dumped_ixps = [] for node in json_ip: if node['ipaddr4'] is None: temp = '' else: temp = node['ipaddr4'] ips = handler.extract_ip(temp, 'IP') for ixpip in ips: ixpip = handler.clean_ip(ixpip, 'IP') if handler.is_valid_ip_address(ixpip, 'IP', 'PDB'): if (ixpip not in ixp_to_asn.keys() and ixpip not in dumped_ixps and ixpip not in reserved_tree and ixpip in temp_subnet_tree): ixp_to_asn[ixpip] = [str(node['asn'])] elif ixpip in ixp_to_asn.keys(): if ixp_to_asn[ixpip] != [str(node['asn'])]: dumped_ixps.append(ixpip) ixp_to_asn.pop(ixpip, None) return ixp_to_asn
def check_edges(self, rule, path_asn, current, str_to_chk, ixp_long, ixp_short): ''' Checks the similarity of the concatenated numbers in case of AS_M and IXP_IP keywords of the border hops of a rule with hop window of size three. Input: a) rule: The current rule. b) path_asn: The AS path. c) current: The current hop in the path. d) str_to_chk: The condition keyword of the rule. e) ixp_long,ixp_short: The long and short IXP names. Output: True if the condition is satisfied, False otherwise. ''' [final1, final2] = self.find_numbers(rule, str_to_chk, current, False) if final1 == '' or final2 == '': return True if self.is_int(final1) and self.is_int(final2) and 'AS_M' in str_chk: if (final1 == final2 and path_asn[current - 1] != path_asn[current + 1]) or ( final1 != final2 and path_asn[current - 1] == path_asn[current + 1]): return False elif self.is_int(final1) and self.is_int( final2) and 'IXP_IP' in str_chk: string_hanlde = string_handler.string_handler() flag = (string_handle.string_comparison(ixp_long[current - 1], ixp_long[current + 1]) or string_handle.string_comparison(ixp_short[current - 1], ixp_short[current + 1])) if (final1 == final2 and not flag) or (final1 != final2 and flag): return False return True
def merge_cc(self, d1, d2): ''' Returns a dictionary C containing the union of the dictionaries A and B. The input dictionaries contain key-to-IXP long, short name entries. The key values can be either IXP Prefix or IP. E.g.: if A[id1]=[a,b], B[id1]=[c,d] then C[id1]=[a,b], if and only if a is similar to c and b is similar to d. [a,b] are the valid IXP long and short names for the IXP subnet. Input: a) d1,d2: The two dictionaries to be merged. Output: a) d1 or d2: The output dictionary after merging. ''' handle = string_handler.string_handler() if len(d2) > len(d1): for k in d1.keys(): if k in d2.keys(): mytuple = self.assign_countries( d1[k][0], d2[k][0], d1[k][1], d2[k][1]) if mytuple != []: d2[k] = mytuple else: d2.pop(k) else: d2[k] = d1[k] return d2 else: for k in d2.keys(): if k in d1.keys(): mytuple = self.assign_countries( d1[k][0], d2[k][0], d1[k][1], d2[k][1]) if mytuple != []: d1[k] = mytuple else: d1.pop(k) else: d1[k] = d2[k] return d1
def merge_keys2names(self, d1, d2): ''' Returns a dictionary C containing the union of the dictionaries A and B. The input dictionaries contain key-to-IXP long, short name entries. The key values can be either IXP Prefix or IP. E.g.: if A[id1]=[a,b], B[id1]=[c,d] then C[id1]=[a,b], if a is similar to c and b is similar to d. [a,b] are the valid IXP long and short names for the IXP subnet. Input: a) d1,d2: The two dictionaries to be merged. Output: a) d1 or d2: The output dictionary after merging. ''' handle = string_handler.string_handler() for k in d1: if k in d2: assign_tuple = [] for IXP1 in d1[k]: for IXP2 in d2[k]: assign_tuple = assign_tuple + \ handle.assign_names( IXP1[1], IXP2[1], IXP1[0], IXP2[0]) deleted = [] for i in range(0, len(assign_tuple) - 1): for j in range(i + 1, len(assign_tuple)): if len(handle.assign_names(assign_tuple[i][0], assign_tuple[j][0], assign_tuple[i][1], assign_tuple[j][1])) == 1: if j not in deleted: deleted = [j] + deleted for node in deleted: del assign_tuple[node] d2[k] = assign_tuple else: d2[k] = d1[k] return d2
def include_additional_prefixes(self, final_sub2name, subTree, additional_subnet2name, final_subnet2country, additional_pfx2cc, help_tree, additional_ixp_ip2asn): ''' Includes the additional info subnet2names to the final dictionaries. Input: a) final_sub2name: The final {IXP Subnet}=[[IXP name long, IXP name short]] dictionary. b) subTree: The final [IXP Subnet]=[IXP name long, IXP name short]. c) additional_subnet2name: The user imported {IXP Subnet}=[[IXP name long, IXP name short]]. d) help_tree: SubneTree containing the IXP Subnets. e) final_subnet2country: subTree: The final [IXP Subnet]=[[IXP name long, IXP name short]]. f) additional_ixp_ip2asn: {IXP IP} = ASN, all the IXP IPs from additional file. Output: a) subTree: The final [IXP Subnet]=[[IXP name long, IXP name short]]. b) final_sub2name: The final dictionary with {IXP Subnet} = [[IXP long name, IXP short name]]. c) final_subnet2country: A dictioanry containing {IXP Subnet} = [ IXP country, IXP city]. ''' handle = string_handler.string_handler() for pfx in additional_subnet2name: # Include and/or overwrite existing PDB & PCH prefixes based on additional prefixes. if handle.sub_prefix_check(pfx, help_tree) and pfx.split('/')[0] not in additional_ixp_ip2asn: subTree.remove(help_tree[pfx]) final_sub2name.pop(help_tree[pfx]) final_subnet2country.pop(help_tree[pfx]) help_tree.remove(help_tree[pfx]) final_sub2name[pfx] = [additional_subnet2name[pfx]] subTree[pfx] = [additional_subnet2name[pfx]] final_subnet2country[pfx] = additional_pfx2cc[pfx] return (subTree, final_sub2name, final_subnet2country)
def merge_cc(self, d1, d2): ''' Returns a dictionary C containing the union of the dictionaries A and B. The input dictionaries contain key-to-IXP long, short name entries. The key values can be either IXP Prefix or IP. E.g.: if A[id1]=[a,b], B[id1]=[c,d] then C[id1]=[a,b], if and only if a is similar to c and b is similar to d. [a,b] are the valid IXP long and short names for the IXP subnet. Input: a) d1,d2: The two dictionaries to be merged. Output: a) d1 or d2: The output dictionary after merging. ''' handle = string_handler.string_handler() if len(d2) > len(d1): big = d2 small = d1 else: big = d1 small = d2 for k in small: if k in big: # Compare countries and cities between the two dictionaries # example entries of dicts {'198.32.118.0/24': ['US', 'New York']} mytuple = self.assign_countries( small[k][0], big[k][0], small[k][1], big[k][1]) if mytuple != []: big[k] = mytuple else: big.pop(k) else: big[k] = small[k] return big
def extract_names(self, json_names, country2cc): ''' Extracts the IXP ID-to-IXP names from the ix.json file. Input: a) json_names: A json table with ix ids and the IXP long and short names. b) country2cc: Country to country code dictionary. Output: a) names_dict: A dictionary with {ix id}=[IXP long name, IXP short name]. b) region_dict: A dictionary with {ix id}=[IXP country,IXP city]. ''' names_dict = {} region_dict = {} handle_string = string_handler.string_handler() for node in json_names: [long_name, short_name] = handle_string.clean_long_short( node['name_long'], node['name']) if (len(long_name) > len(short_name)): names_dict[node['id']] = [long_name, short_name] else: names_dict[node['id']] = [short_name, long_name] country = re.sub('([^\s\w]|_)+', ' ', node['country'].strip()) country = ' '.join(self.unique_list(country.split())) country = re.sub(' +', ' ', country) city = re.sub('([^\s\w]|_)+', ' ', node['city'].strip()) city = ' '.join(self.unique_list(city.split())) city = re.sub(' +', ' ', city) try: region_dict[node['id']] = [country2cc[country], city] except KeyError: region_dict[node['id']] = [country, city] return (names_dict, region_dict)
def extract_ip(self, json_ip, temp_subnet_tree, add_subnet_tree, reserved_tree): ''' Extracts the IXP IPs from peeringdb. Input: a) json_IP: A json table containing IXP IPs, IXP short names and IXP IDs. b) temp_subnet_Tree: The Subnet Tree containing the IXP subnets from peeringdb. c) add_subnet_tree: The SubnetTree containing the user input on IXP subnets from additional_info. d) reserved_tree: The SubnetTree containing the reserved Subnets. Output: a) ixp_to_asn: A dictionary with {IXP IP}=[ASN]. ''' handler = string_handler.string_handler() ixp_to_asn = {} dumped_ixps = [] for node in json_ip: if node['ipaddr4'] is None: temp = '' else: temp = node['ipaddr4'] ips = handler.extract_ip(temp, 'IP') for ixpip in ips: if handler.is_valid_ip_address(ixpip, 'IP'): if (ixpip not in ixp_to_asn.keys() and ixpip not in dumped_ixps and ixpip not in reserved_tree and ixpip in temp_subnet_tree): ixp_to_asn[ixpip] = [str(node['asn'])] elif ixpip in ixp_to_asn.keys(): if ixp_to_asn[ixpip] != [str(node['asn'])]: dumped_ixps.append(ixpip) ixp_to_asn.pop(ixpip, None) return ixp_to_asn
def check_names(self, rule, expression, current, i, encounter_type, str_to_chk, ixp_long, ixp_short): ''' Checks the similarity of the concatenated numbers in case of IXP_IP keyword for consecutive IXP IPs in the path. It also compares the IXP short and long names. Input: a) rule: The current rule. b) expression: The current condition of the condition part of the rule. c) i: The hop of the current condition in the rule. d) encounter_type: The resolved IP path based on the encountered types (IXP IP, IXP Prefix, Normal IP, Illegal IP). e) str_to_chk: The condition keyword of the rule. d) ixp_long,ixp_short: The IXP long and short names. Output: True if the condition is satisfied, False otherwise. ''' string_handle = string_handler.string_handler() if len(rule) > i + 1 and len(ixp_long) > current + 1: [final1, final2] = self.find_numbers(rule, str_to_chk, current, True) if final1 == '' or final2 == '': return True if self.is_int(final1) and self.is_int(final2): flag = (string_handle.string_comparison( ixp_long[current], ixp_long[current + 1]) or string_handle.string_comparison( ixp_short[current], ixp_short[current + 1])) if (final1 == final2 and not flag) or (final1 != final2 and flag): return False return True
def pch_handle_ixpm(self, reserved_tree, add_info_tree): ''' Extracts the IXP IPs from PCH. Input: a) filename: The ixp_membership.csv file. b) self.homepath: The directory path of the database. c) reserved_tree: The SubnetTree containing the reserved Subnets. d) add_info_tree: The SubnetTree containing user imported IXP prefixes. Output: a) ixpip2asn: A dictionary with {IXP IP}=[ASN]. ''' doc = self.file_opener(self.filename_ixp_membership, 3) tree = SubnetTree.SubnetTree() ixpip2asn = {} sub_to_ixp = {} hstring = string_handler.string_handler() dumped_ixps = [] # Skip the first line of the file. next(doc) for line in doc: temp_string = line.split(',') if len(temp_string) > 1: ip = hstring.extract_ip(temp_string[1], 'IP') for inode in ip: inode = hstring.clean_ip(inode, 'IP') if hstring.is_valid_ip_address( inode, 'IP', 'PCH') and temp_string[3].replace(' ', '') != '': subnet = hstring.extract_ip(temp_string[0], 'Subnet') for snode in subnet: snode = hstring.clean_ip(snode, 'Subnet') if hstring.is_valid_ip_address( snode, 'Subnet', 'PCH'): tree[snode] = snode if (inode in tree and inode not in ixpip2asn.keys() and inode not in dumped_ixps and inode not in reserved_tree) or ( inode in add_info_tree): ixpip2asn[inode] = [ temp_string[3].replace(' ', '') ] elif inode in ixpip2asn.keys(): if ixpip2asn[inode] != [ temp_string[3].replace(' ', '') ]: ixpip2asn.pop(inode, None) dumped_ixps.append(inode) tree.remove(snode) doc.close() return ixpip2asn
def print_traIXroute_dest(self, traixparser, db_extract, dns_print, dst_ip, src_ip='', info=''): ''' Prints traIXroute destination. Input: a) dns_print: Flag to enable resolving IP or FQDN. b) dst_ip: The destination IP/FQDN to probe. c) src_ip: The IP that issued the probe (optional). d) info: Traceroute path description. ''' string_handle = string_handler.string_handler() print_data = 'traIXroute' dns_name = '*' output_IP = '*' if string_handle.is_valid_ip_address(dst_ip, 'IP', 'CLI'): if dns_print: try: dns_name = socket.gethostbyaddr(dst_ip)[0] except: pass output_IP = dst_ip else: if dns_print: try: output_IP = socket.gethostbyname(dst_ip) except: pass dns_name = dst_ip if src_ip != '': origin_dns = '*' if dns_print: try: origin_dns = socket.gethostbyaddr(src_ip)[0] except: pass print_data += ' from ' + origin_dns + ' (' + src_ip + ')' if traixparser.flags['asn']: if src_ip in db_extract.asn_routeviews: print_data += ' AS'+db_extract.asn_routeviews[src_ip] elif src_ip: print_data += ' AS*' print_data += ' to ' + dns_name + ' (' + output_IP + ')' if traixparser.flags['asn']: if output_IP in db_extract.asn_routeviews: print_data += ' AS'+db_extract.asn_routeviews[output_IP] else: print_data += ' AS*' if info != '': print_data += ' info: ' + info self.measurement_info += print_data + '\n'
def routeviews_extract(self, reserved_sub_tree): ''' Imports the file from routeviews to a Subnet Tree. Input: a) filename: The file name from routeviews to parse. b) mypath: The directory path of the database folder. c) reserved_sub_tree: The SubnetTree containing the reserved Subnets. d) config: Dictionary that contains the config file. e) db_path: The path to the Database directory. Output: a) Stree: A SubnetTree containing IXP Subnet-to-ASNs. b) temp_dict: A dictionary containing {IXP Subnet} = AS. ''' handler = string_handler.string_handler() try: f = open(self.homepath + '/database' + self.route_filename) except: print(self.route_filename + ' was not found.') if not self.downloader.download_routeviews(): print("Could not download " + self.route_filename + ". Copying from the default database.") try: copyfile( self.libpath + '/database/Default' + self.route_filename, self.homepath + '/database' + self.route_filename) except: print('Could not copy ' + self.route_filename + ' from the default database.') try: f = open(self.homepath + '/database' + self.route_filename) except: print('Could not open ' + self.route_filename + '. Exiting.') sys.exit(0) temp_dict = {} Stree = SubnetTree.SubnetTree() for line in f: temp = line.split() myip = handler.extract_ip(temp[0], 'IP') if len(myip) > 0: if handler.is_valid_ip_address(myip[0] + '/' + str(temp[1]), 'Subnet'): if not handler.sub_prefix_check( myip[0] + '/' + str(temp[1]), reserved_sub_tree): Stree[myip[0] + '/' + temp[1]] = temp[2] temp_dict[myip[0] + '/' + temp[1]] = temp[2] f.close() return Stree, temp_dict
def extract_pfx(self, json_pfx, ixlan_dict, id_to_names, reserved_tree, region_dict): ''' Extracts the prefixes from ixpfxs: Input: a) json_pfx: A json table containing the IXP prefixes and ids to ixlan. b) ixlan_dict: A dictionary with {ixlan id}=[ix id] to bind IXP prefixes and IXP names. c) id_to_names: A dictionary with {ix id}=[IXP long name, IXP short name] d) reserved_tree: The SubnetTree containing the reserved Subnets. e) region_dict: A dictionary with {ix id}=[IXP country,IXP city]. Output: a) pfxs_dict: A dictionary with {IXP Subnet}=[IXP long name, IXP short name]. b) temp_subnet_tree: A Subnet Tree with IXP Subnet-to-ix id. c) subnet2region: A dictionary with {IXP Subnet}=[IXP country,IXP city]. ''' handler = string_handler.string_handler() temp_subnet_tree = SubnetTree.SubnetTree() pfxs_dict = {} subnet2region = {} for node in json_pfx: for subnet in handler.extract_ip(node['prefix'], 'Subnet'): subnet = handler.clean_ip(subnet, 'Subnet') if handler.is_valid_ip_address(subnet, 'Subnet', 'PDB'): ixlan_id = node['ixlan_id'] if ixlan_id in ixlan_dict: ix_id = ixlan_dict[ixlan_id] # Checking if there exists IXP name for the given prefix and if the candidate IXP prefix is already inside the dictionary that aggregates the PDB prefixes if ix_id in id_to_names and subnet not in pfxs_dict: if id_to_names != ['', '']: pfxs_dict[subnet] = [id_to_names[ix_id]] else: continue temp_subnet_tree[subnet] = [id_to_names[ix_id]] subnet2region[subnet] = region_dict[ix_id] elif subnet in pfx_dict: assign_tuple = [] for IXP in pfx_dict[subnet]: assign_tuple = assign_tuple + \ handler.assign_names(IXP[1], id_to_names[ix_id][1], IXP[0], id_to_names[ix_id][0]) pfxs_dict[subnet] = assign_tuple subnet2regions[subnet] = region_dict[ix_id] return (pfxs_dict, temp_subnet_tree, subnet2region)
def extract_pfx(self, json_pfx, ixlan_dict, id_to_names, reserved_tree, region_dict): ''' Extracts the prefixes from ixpfxs: Input: a) json_pfx: A json table containing the IXP prefixes and ids to ixlan. b) ixlan_dict: A dictionary with {ixlan id}=[ix id] to bind IXP prefixes and IXP names. c) id_to_names: A dictionary with {ix id}=[IXP long name, IXP short name] d) reserved_tree: The SubnetTree containing the reserved Subnets. e) region_dict: A dictionary with {ix id}=[IXP country,IXP city]. Output: a) pfxs_dict: A dictionary with {IXP Subnet}=[IXP long name, IXP short name]. b) temp_subnet_tree: A Subnet Tree with IXP Subnet-to-ix id. c) subnet2region: A dictionary with {IXP Subnet}=[IXP country,IXP city]. ''' handler = string_handler.string_handler() temp_subnet_tree = SubnetTree.SubnetTree() pfxs_dict = {} subnet2region = {} for node in json_pfx: for subnet in handler.extract_ip(node['prefix'], 'Subnet'): subnet = handler.clean_ip(subnet, 'Subnet') if handler.is_valid_ip_address(subnet, 'Subnet', 'PDB'): ixlan_id = node['ixlan_id'] if ixlan_id in ixlan_dict: ix_id = ixlan_dict[ixlan_id] # Checking if there exists IXP name for the given prefix and if the candidate IXP prefix is already inside the dictionary that aggregates the PDB prefixes if ix_id in id_to_names and subnet not in pfxs_dict: if id_to_names != ['', '']: pfxs_dict[subnet] = [id_to_names[ix_id]] else: continue temp_subnet_tree[subnet] = [id_to_names[ix_id]] subnet2region[subnet] = region_dict[ix_id] elif subnet in pfx_dict: assign_tuple = [] for IXP in pfx_dict[subnet]: assign_tuple = assign_tuple + \ handler.assign_names(IXP[1], id_to_names[ix_id][1], IXP[0], id_to_names[ix_id][0]) pfxs_dict[subnet] = assign_tuple subnet2regions[subnet] = region_dict[ix_id] return (pfxs_dict, temp_subnet_tree, subnet2region)
def routeviews_extract(self, reserved_sub_tree): ''' Imports the file from routeviews to a Subnet Tree. Input: a) filename: The file name from routeviews to parse. b) mypath: The directory path of the database folder. c) reserved_sub_tree: The SubnetTree containing the reserved Subnets. d) config: Dictionary that contains the config file. e) db_path: The path to the Database directory. Output: a) Stree: A SubnetTree containing IXP Subnet-to-ASNs. b) temp_dict: A dictionary containing {IXP Subnet} = AS. ''' handler = string_handler.string_handler() try: f = open(self.homepath + '/database' + self.route_filename) except: print(self.route_filename + ' was not found.') if not self.downloader.download_routeviews(): print("Could not download " + self.route_filename + ". Copying from the default database.") try: copyfile(self.libpath + '/database/Default' + self.route_filename, self.homepath + '/database' + self.route_filename) except: print('Could not copy ' + self.route_filename + ' from the default database.') try: f = open(self.homepath + '/database' + self.route_filename) except: print('Could not open ' + self.route_filename + '. Exiting.') sys.exit(0) temp_dict = {} Stree = SubnetTree.SubnetTree() for line in f: temp = line.split() myip = handler.extract_ip(temp[0], 'IP') if len(myip): if handler.is_valid_ip_address(myip[0] + '/' + temp[1], 'Subnet', 'RouteViews'): if not handler.sub_prefix_check(myip[0] + '/' + temp[1], reserved_sub_tree): Stree[myip[0] + '/' + temp[1]] = temp[2] temp_dict[myip[0] + '/' + temp[1]] = temp[2] f.close() return Stree, temp_dict
def trace_call(self, IP_name, classic, arguments): ''' Calls scamper and extracting the route from the text file. Input: a) IP_name: The IP to probe. b) classic: Flag to enable scamper or traceroute. c) arguments: The arguments of the selected tool. Output: a) route: The IP path list. b) mytime: The list with the hop delays. ''' # A domain name has been given as destination to send the probe instead # of an IP address. string_handle = string_handler.string_handler() arguments = arguments.strip() if not string_handle.is_valid_ip_address(IP_name, 'IP'): try: IP_name = socket.gethostbyname(IP_name) except: print( '--> ' + IP_name + ' has wrong address format. Expected an IPv4 format or a valid url.' ) return [], [] elif not string_handle.check_input_ip(IP_name): print( '--> ' + IP_name + ' has wrong address format. Expected an IPv4 format or a valid url.' ) return [], [] if classic: [route, mytime] = self.scamper_call(IP_name, arguments) if not len(route): print( '--> Scamper returned an empty IP path. You may use "sudo" in the beginning or the scamper arguments might be wrong.' ) elif not classic: [route, mytime] = self.traceroute_call(IP_name, arguments) if not len(route): print( '--> Traceroute returned an empty IP path. You may use "sudo" in the beginning.' ) return [route, mytime]
def pch_handle_ixpm(self, reserved_tree, add_info_tree): ''' Extracts the IXP IPs from PCH. Input: a) filename: The ixp_membership.csv file. b) self.homepath: The directory path of the database. c) reserved_tree: The SubnetTree containing the reserved Subnets. d) add_info_tree: The SubnetTree containing user imported IXP prefixes. Output: a) ixpip2asn: A dictionary with {IXP IP}=[ASN]. ''' doc = self.file_opener(self.filename_ixp_membership, 3) tree = SubnetTree.SubnetTree() ixpip2asn = {} sub_to_ixp = {} hstring = string_handler.string_handler() dumped_ixps = [] # Skip the first line of the file. next(doc) for line in doc: temp_string = line.split(',') if len(temp_string) > 1: ip = hstring.extract_ip(temp_string[1], 'IP') for inode in ip: inode = hstring.clean_ip(inode, 'IP') if hstring.is_valid_ip_address(inode, 'IP', 'PCH') and temp_string[3].replace(' ', '') != '': subnet = hstring.extract_ip(temp_string[0], 'Subnet') for snode in subnet: snode = hstring.clean_ip(snode, 'Subnet') if hstring.is_valid_ip_address(snode, 'Subnet','PCH'): tree[snode] = snode if (inode in tree and inode not in ixpip2asn.keys() and inode not in dumped_ixps and inode not in reserved_tree) or (inode in add_info_tree): ixpip2asn[inode] = [ temp_string[3].replace(' ', '')] elif inode in ixpip2asn.keys(): if ixpip2asn[inode] != [temp_string[3].replace(' ', '')]: ixpip2asn.pop(inode, None) dumped_ixps.append(inode) tree.remove(snode) doc.close() return ixpip2asn
def extract_pfx(self, json_pfx, ixlan_dict, id_to_names, reserved_tree, region_dict): ''' Extracts the prefixes from ixpfxs: Input: a) json_pfx: A json table containing the IXP prefixes and ids to ixlan. b) ixlan_dict: A dictionary with {ixlan id}=[ix id] to bind IXP prefixes and IXP names. c) id_to_names: A dictionary with {ix id}=[IXP long name, IXP short name] d) reserved_tree: The SubnetTree containing the reserved Subnets. e) region_dict: A dictionary with {ix id}=[IXP country,IXP city]. Output: a) pfxs_dict: A dictionary with {IXP Subnet}=[IXP long name, IXP short name]. b) temp_subnet_tree: A Subnet Tree with IXP Subnet-to-ix id. c) subnet2region: A dictionary with {IXP Subnet}=[IXP country,IXP city]. ''' handler = string_handler.string_handler() pfxs_dict = {} i = 0 temp_subnet_tree = SubnetTree.SubnetTree() subnet2region = {} for node in json_pfx: subnet = handler.extract_ip(node['prefix'], 'Subnet') for s in subnet: if handler.is_valid_ip_address(s, 'Subnet'): ixpfx = s ixlan_id = node['ixlan_id'] if ixlan_id in ixlan_dict.keys(): ix_id = ixlan_dict[ixlan_id] if ix_id in id_to_names.keys() and s not in pfxs_dict: if id_to_names != ['', '']: pfxs_dict[s] = [id_to_names[ix_id]] else: continue temp_subnet_tree[s] = [id_to_names[ix_id]] subnet2region[s] = region_dict[ix_id] elif s in pfx_dict: assign_tuple = [] for IXP in pfx_dict: assign_tuple = assign_tuple + \ handler.assign_names(IXP[1], id_to_names[ix_id][ 1], IXP[0], id_to_names[ix_id][0]) pfxs_dict[s] = assign_tuple subnet2regions[s] = region_dict[ix_id] return (pfxs_dict, temp_subnet_tree, subnet2region)
def assign_countries(self, country1, country2, city1, city2): ''' Takes as inputs countries and cities retrieved by PCH and PeeringDB assigned to a certain prefix and selects the countries and the cities to keep. Input: a) country1,country2: The country names assigned to an IXP Prefix (from pch and peeringDB). b) city1,city2: The city names assigned to an IXP Prefix (from pch and peeringDB). Output: a) d3: The [country,city] list after merging. ''' string_handle = string_handler.string_handler() country = '' city = '' if string_handle.string_comparison(country1, country2): country = country1 elif country1 == '' and country2 != '': country = country2 elif country1 != '' and country2 == '': country = country1 if string_handle.string_comparison(city1, city2): city = city2 elif city1 == '' and city2 != '': city = city2 elif city1 != '' and city2 == '': city = city1 d3 = [] if country == '': country = country1 + '//' + country2 if city == '': if string_handle.shortinlong(city1, city2): city = city2 elif string_handle.shortinlong(city2, city1): city = city1 else: city = city1 + '//' + city2 if country != '' and country != '': d3 = [country, city] return d3
def assign_countries(self, country1, country2, city1, city2): ''' Takes as inputs countries and cities retrieved by PCH and PeeringDB assigned to a certain prefix and selects the countries and the cities to keep. Input: a) country1,country2: The country names assigned to an IXP Prefix (from pch and peeringDB). b) city1,city2: The city names assigned to an IXP Prefix (from pch and peeringDB). Output: a) The [country,city] list after merging. ''' string_handle = string_handler.string_handler() country = '' city = '' # Comparing IXP countries. if string_handle.string_comparison(country1, country2): country = country1 elif country1 == '' and country2 != '': country = country2 elif country1 != '' and country2 == '': country = country1 # Both countries are different if country == '': country = country1 + '//' + country2 # Comparing IXP cities. if string_handle.string_comparison(city1, city2): city = city2 elif city1 == '' and city2 != '': city = city2 elif city1 != '' and city2 == '': city = city1 # Both cities are different if city == '': if string_handle.shortinlong(city1, city2): city = city2 elif string_handle.shortinlong(city2, city1): city = city1 else: city = city1 + '//' + city2 return [country, city] if country != '' and city != '' else []
def exclude_reserved_subpref(self, subTree, final_sub2name, reserved_list, final_subnet2country): ''' Excludes the reserved subprefixes. Input: a) subTree: The SubnetTree containing [[IXP long name, IXP short name]]. b) final_sub2name: The dictionary with {IXP Subnet} = [[IXP long name, IXP short name]]. c) reserved_list: A list containing the reserved prefixes. d) final_subnet2country: A dictioanry containing {IXP Subnet} = [ IXP country, IXP city]. Output: a) subTree: The SubnetTree containing [[IXP long name, IXP short name]]. b) final_sub2name: The cleaned dictionary with {IXP Subnet} = [[IXP long name, IXP short name]]. ''' handle_string = string_handler.string_handler() for prefix in reserved_list: if handle_string.sub_prefix_check(prefix, subTree): rprefix = subTree[prefix] subTree.remove(rprefix) final_sub2name.pop(rprefix) final_subnet2country.pop(rprefix) return (subTree, final_sub2name)
def pch_handle_long(self, country2cc): ''' Returns a dictionary with the IXP long names. Input: a) filename: The ixp_exchange.csv file. b) self.homepath: The directory path of the database. c) country2cc: Dictionary that contains country names to country codes. Output: a) ixpip2long: A dictionary with {keyid}=[IXP long name]. b) IXP_region: A dictionary with {keyid}=[IXP country, IXP city]. ''' handle_string = string_handler.string_handler() doc = self.file_opener(self.filename_excha, 2) ixpip2long = {} IXP_region = {} # Skip the first line of the file. next(doc) for line in doc: temp_string = [item.strip() for item in line.split(', ')] if len(temp_string) > 6: if handle_string.string_comparison(temp_string[6], 'Active'): # IXP long name ixpip2long[temp_string[0]] = temp_string[4] # Country name country = handle_string.format_country_city(temp_string[2]) # City name city = handle_string.format_country_city(temp_string[3]) try: IXP_region[temp_string[0]] = country2cc[country], city except KeyError: print('Warning, update your CCs for:', country) IXP_region[temp_string[0]] = country, city doc.close() return (ixpip2long, IXP_region)
def asn_memb(self, IXP_final, sub2names): ''' Constructs a new dictionary with {ASN}={IXP long name, IXP short name} based on IXP_final and dirty_ixp2asn dictionaries. Input: a) IXP_final: A dictionary with {IXP}=[ASN] after merging peeringdb and pch datasets. c) sub2names: The SubnetTree with the IXP prefixes to a list of [IXP long name, short name]. Output: a) ASn_memb: A dictionary with {ASN}=[IXP long name, IXP short name]. ''' stringh = string_handler.string_handler() ASn_memb = {} for node in IXP_final.keys(): new_key = IXP_final[node][0] temp_string_node = sub2names[node] if new_key not in ASn_memb.keys(): ASn_memb[new_key] = [] for IXP in temp_string_node: ASn_memb[new_key].append(IXP) return ASn_memb
def pch_handle_long(self, country2cc): ''' Returns a dictionary with the IXP long names. Input: a) filename: The ixp_exchange.csv file. b) self.homepath: The directory path of the database. c) country2cc: Dictionary that contains country names to country codes. Output: a) ixpip2long: A dictionary with {keyid}=[IXP long name]. b) IXP_region: A dictionary with {keyid}=[IXP country, IXP city]. ''' doc = self.file_opener(self.filename_excha, 2) ixpip2long = {} IXP_region = {} flag = True handled_string = string_handler.string_handler() for line in doc: if flag: flag = False continue temp_string = line.split(',') if len(temp_string) > 6: if handled_string.string_comparison(temp_string[5], 'Active'): ixpip2long[temp_string[0]] = temp_string[4] country = re.sub('([^\s\w]|_)+', ' ', temp_string[2].strip()) country = ' '.join(self.unique_list(country.split(' '))) country = re.sub(' +', ' ', country) city = re.sub('([^\s\w]|_)+', ' ', temp_string[3].strip()) city = ' '.join(self.unique_list(city.split(' '))) city = re.sub(' +', ' ', city) try: IXP_region[temp_string[0]] = country2cc[country], city except KeyError: IXP_region[temp_string[0]] = country, city return (ixpip2long, IXP_region)
def pch_handle_long(self, country2cc): ''' Returns a dictionary with the IXP long names. Input: a) filename: The ixp_exchange.csv file. b) self.homepath: The directory path of the database. c) country2cc: Dictionary that contains country names to country codes. Output: a) ixpip2long: A dictionary with {keyid}=[IXP long name]. b) IXP_region: A dictionary with {keyid}=[IXP country, IXP city]. ''' handle_string = string_handler.string_handler() doc = self.file_opener(self.filename_excha, 2) ixpip2long = {} IXP_region = {} # Skip the first line of the file. next(doc) for line in doc: temp_string = [item.strip() for item in line.split(', ')] if len(temp_string) > 6: if handle_string.string_comparison(temp_string[6], 'Active'): # IXP long name ixpip2long[temp_string[0]] = temp_string[4] # Country name country = handle_string.format_country_city(temp_string[2]) # City name city = handle_string.format_country_city(temp_string[3]) try: IXP_region[temp_string[0]] = country2cc[country], city except KeyError: print('Warning, update your CCs for:', country) IXP_region[temp_string[0]] = country, city doc.close() return (ixpip2long, IXP_region)
def print_traIXroute_dest(self, dst_ip, src_ip='', info=''): ''' Prints traIXroute destination. Input: a) dst_ip: The destination IP/FQDN to probe. b) src_ip: The IP that issued the probe (optional). c) info: Traceroute path description. ''' string_handle = string_handler.string_handler() dns_name = '*' output_IP = '*' if string_handle.is_valid_ip_address(dst_ip, 'IP'): try: dns_name = socket.gethostbyaddr(dst_ip)[0] except: pass output_IP = dst_ip else: try: output_IP = socket.gethostbyname(dst_ip) except: pass dns_name = dst_ip if src_ip != '': origin_dns = '*' try: origin_dns = socket.gethostbyaddr(src_ip)[0] except: pass src_ip = ' from ' + origin_dns + ' (' + src_ip + ')' print_data = 'traIXroute' + src_ip + ' to ' + \ dns_name + ' (' + output_IP + ').' if info != '': print_data = print_data + ' info: ' + info self.measurement_info += print_data + '\n'
def extract_names(self, json_names, country2cc): ''' Extracts the IXP ID-to-IXP names from the ix.json file. Input: a) json_names: A json table with ix ids and the IXP long and short names. b) country2cc: Country to country code dictionary. Output: a) names_dict: A dictionary with {ix id}=[IXP long name, IXP short name]. b) region_dict: A dictionary with {ix id}=[IXP country,IXP city]. ''' names_dict = {} region_dict = {} handle_string = string_handler.string_handler() for node in json_names: [long_name, short_name ] = handle_string.clean_long_short(node['name_long'], node['name']) if long_name == '': long_name = short_name elif short_name == '': short_name = long_name if (len(long_name) > len(short_name)): names_dict[node['id']] = [long_name, short_name] else: names_dict[node['id']] = [short_name, long_name] # Country Code country = handle_string.format_country_city(node['country']) country = country2cc[country] if country in country2cc else country # City Name city = handle_string.format_country_city(node['city']) region_dict[node['id']] = [country, city] return (names_dict, region_dict)
def extract_names(self, json_names, country2cc): ''' Extracts the IXP ID-to-IXP names from the ix.json file. Input: a) json_names: A json table with ix ids and the IXP long and short names. b) country2cc: Country to country code dictionary. Output: a) names_dict: A dictionary with {ix id}=[IXP long name, IXP short name]. b) region_dict: A dictionary with {ix id}=[IXP country,IXP city]. ''' names_dict = {} region_dict = {} handle_string = string_handler.string_handler() for node in json_names: [long_name, short_name] = handle_string.clean_long_short( node['name_long'], node['name']) if long_name == '': long_name = short_name elif short_name == '': short_name = long_name if (len(long_name) > len(short_name)): names_dict[node['id']] = [long_name, short_name] else: names_dict[node['id']] = [short_name, long_name] # Country Code country = handle_string.format_country_city(node['country']) country = country2cc[country] if country in country2cc else country # City Name city = handle_string.format_country_city(node['city']) region_dict[node['id']] = [country, city] return (names_dict, region_dict)
def pch_handle_sub(self, reserved_tree, long_mem, IXP_region): ''' Extracts the IXP keys-to-Subnets from the ixp_subnets.csv file. Input: a) filename: The ixp_subnets.csv file. b) self.homepath: The directory path of the database. c) reserved_tree: The SubnetTree containing the reserved Subnets. d) long_mem: Dictionary containing id to long names. e) IXP_region: Dictionary containing {IXP Subnet} = [IXP country, IXP city]. Output: a) subnets: A dictionary with {IXP Subnet}=[IXP long name, IXP short name]. b) IXP_cc: A dictionary with {IXP Subnet}=[IXP country,IXP region]. c) Subnet_names: SubnetTree with {Subnet} = [IXP long name, IXP short name]. ''' handled_string = string_handler.string_handler() doc = self.file_opener(self.filename_subset, 1) IXP_cc = {} subnets = {} Subnet_names = SubnetTree.SubnetTree() # Skip the first line of the file. next(doc) for line in doc: temp_string = line.split(',') if len(temp_string) > 5: mykey = temp_string[0] myip = handled_string.extract_ip(temp_string[5], 'Subnet') for ips in myip: # Clean misspelled Subnets. ips = handled_string.clean_ip(ips, 'Subnet') if ips != '' and handled_string.string_comparison(temp_string[2], 'Active'): if handled_string.is_valid_ip_address(ips, 'Subnet', 'PCH') and ips not in subnets: if mykey in long_mem.keys(): IXP_cc[ips] = IXP_region[mykey] [long_name, short_name] = handled_string.clean_long_short( long_mem[mykey], temp_string[1]) if long_name == '': long_name = short_name elif short_name == '': short_name = long_name if len(long_name) > len(short_name): subnets[ips] = [[long_name, short_name]] else: subnets[ips] = [[short_name, long_name]] elif temp_string[1] != '': [long_name, short_name] = handled_string.clean_long_short("", temp_string[1]) if len(short_name) < len(long_name): subnets[ips] = [['', short_name]] else: subnets[ips] = [['', long_name]] else: continue elif ips in subnets: IXP_cc[ips] = IXP_region[mykey] [long_name, short_name] = handled_string.clean_long_short( long_mem[mykey], temp_string[1]) if short_name > long_name: tmp_name_string = long_name long_name = short_name short_name = tmp_name_string assigned_tuple = [] for IXP in subnets[ips]: assigned_tuple = assigned_tuple + \ handled_string.assign_names( IXP[0], short_name, IXP[1], long_name) subnets[ips] = assigned_tuple if ips in subnets: Subnet_names[ips] = subnets[ips] doc.close() return (subnets, IXP_cc, Subnet_names)
def check_rules(self, path, rule, path_asn, path_cur, ixp_long, ixp_short, asn2names, encounter_type): ''' Checks if the condition part of a rule is satisfied. Input: a) rule: The condition part of the candidate IXP detection rule. b) path_asn: The AS path. c) path_cur: The current hop in the path. d) ixp_long, ixp_short: The long and short IXP names. e) asn2names: A dictionary with a list of lists of short and long IXP names in which an AS is member - {ASN}=[[name long,name short],[name long,name short]...]. f) encounter_type: The resolved IP path based on the encountered types (IXP IP, IXP Prefix, Normal IP, Illegal IP). g) path: The IP path. Output: True if the expression is satisfied, False otherwise. ''' if len(rule) > len(path_asn): return False for i,item in enumerate(rule): if len(path_asn) > path_cur + i - 1: if ('IXP_IP' in item and '!AS_M' in item) and 'IXP prefix' not in encounter_type[path_cur + i - 1] and 'IXP IP' not in encounter_type[path_cur + i - 1]: return False elif 'IXP_IP' in item and 'AS_M' in item and '!' not in item and 'IXP IP' not in encounter_type[path_cur + i - 1]: return False elif ('IXP_IP' not in item or '!AS_M' not in item) and 'IXP prefix' in encounter_type[path_cur + i - 1]: return False elif ('IXP_IP' not in item or 'AS_M' not in item) and 'IXP IP' in encounter_type[path_cur + i - 1]: return False # Applies each condition of the condition part of the candidate rule onto the path. string_h = string_handler.string_handler() check = 0 for i,expression in enumerate(rule): # The current condition of the condition part of the rule. current = path_cur + i - 1 # Checking for IXP membership based on a non-IXP IP. if '!AS_M' in expression and 'and' not in expression and path_cur != current: # Finds the path_asn in the routeview path_asn dict. If not, an # assessment is not possible. check += 1 if path_asn[current] == '*' and encounter_type[current] != 'IXP prefix': return False if encounter_type[path_cur] == 'IXP IP' or encounter_type[path_cur] == 'IXP prefix': as_names = '' if path_asn[current] in asn2names.keys(): as_names = asn2names[path_asn[current]] ix_long = ixp_long[path_cur] ix_short = ixp_short[path_cur] else: as_names = '' if path_asn[path_cur] in asn2names.keys(): as_names = asn2names[path_asn[path_cur]] ix_long = ixp_long[current] ix_short = ixp_short[current] for node in as_names: for name in node: if (string_h.string_comparison(ix_long, name) or string_h.string_comparison(ix_short, name)): return False if not self.check_number(rule, expression, path_asn, current, i, encounter_type, '!AS_M'): return False elif 'AS_M' in expression and 'and' not in expression and path_cur != current: if path_asn[current] == '*' and encounter_type[current] != 'IXP prefix': return False check += 1 flag = False if encounter_type[path_cur] == 'IXP IP' or encounter_type[path_cur] == 'IXP prefix': as_names = '' if path_asn[current] in asn2names.keys(): as_names = asn2names[path_asn[current]] ix_long = ixp_long[path_cur] ix_short = ixp_short[path_cur] else: as_names = '' if path_asn[path_cur] in asn2names.keys(): as_names = asn2names[path_asn[path_cur]] ix_long = ixp_long[current] ix_short = ixp_short[current] for node in (as_names): for name in node: if string_h.string_comparison(ix_long, name) or string_h.string_comparison(ix_short, name): flag = 1 break if not flag: return False if not self.check_number(rule, expression, path_asn, current, i, encounter_type, 'AS_M'): return False if '*' in expression and path[current] != '*': return False # Checking for IXP IP or Prefix based on either IXP membership or # Prefixes data. if 'IXP_IP' in expression and '!AS_M' in expression: check += 1 if not self.check_names(rule, expression, current, i, encounter_type, 'IXP_IP', ixp_long, ixp_short): return False elif not self.check_number(rule, expression, path_asn, current, i, encounter_type, '!AS_M'): return False elif 'IXP_IP' in expression and 'AS_M' in expression: check += 1 if not self.check_names(rule, expression, current, i, encounter_type, 'IXP_IP', ixp_long, ixp_short): return False elif not self.check_number(rule, expression, path_asn, current, i, encounter_type, 'AS_M'): return False if len(rule) > 2 and len(path_asn) > current + 1: check += 1 if not self.check_edges(rule, path_asn, current, 'AS_M', ixp_long, ixp_short): return False elif not self.check_edges(rule, path_asn, current, 'IXP_IP', ixp_long, ixp_short): return False if check: return True else: return False
def Subnet_tree(self, Sub, additional_tree, reserved_sub_tree, final_subnet2country): ''' Returns a Subnet Tree containing all the IXP subnets. Input: a) Sub: A dictionary with {Subnet}=[IXP long name,IXP short name] after merging PDB and PCH datasets. b) additional_tree: a Subnet Tree containing all the user {IXP Subnet}=IXP Subnet. c) reserved_sub_tree: A SubnetTree containing the reserved Subnets. d) final_subnet2country: A dictionary with {Subnet}=[country,city]. Output: a) Stree: A Subnet Tree with the IXP names. b) Sub: A dictionary with {Subnet}=[IXP long name,IXP short name]. c) help_tree: A Subnet Tree with IXP Subnets. ''' Stree = SubnetTree.SubnetTree() help_tree = SubnetTree.SubnetTree() handle_string = string_handler.string_handler() for subnet in sorted(Sub): check = handle_string.sub_prefix_check(subnet, help_tree) # If a subprefix does not exist and it is not a reserved prefix. if not check and not handle_string.sub_prefix_check(subnet, additional_tree) and not handle_string.sub_prefix_check(subnet, reserved_sub_tree): Stree[subnet] = Sub[subnet] help_tree[subnet] = subnet # If a subprefix exists. elif check: # Keep initial tuple to find the corresponding prefix in order to keep only prefix in the last step. initial_tuple = Stree[subnet] # Find the prefix that maps to the initial tuple in order to keep the prefix, update the IXP names and delete subprefix in the last step. prefix = [temp_subnet for temp_subnet, temp_ixp_name in Sub.items() if temp_ixp_name == initial_tuple][0] # Gather the IXP names of existing subprefixes in the Subnet Tree. assign_tuple = [] for IXP1 in Stree[subnet]: for IXP2 in Sub[subnet]: assign_tuple = assign_tuple + handle_string.assign_names( IXP1[1], IXP2[1], IXP1[0], IXP2[0]) # Delete similar IXP names assigned to the same prefixes. deleted = [] for i in range(0, len(assign_tuple) - 1): for j in range(i + 1, len(assign_tuple)): if len(handle_string.assign_names(assign_tuple[i][0], assign_tuple[j][0], assign_tuple[i][1], assign_tuple[j][1])) == 1: if j not in deleted: deleted = [j] + deleted for node in deleted: del assign_tuple[node] # Delete Subprefixes when there are Prefixes with same IXP names. if assign_tuple == Stree[subnet]: Sub.pop(subnet) final_subnet2country.pop(subnet) # Keep Prefixes (as dirty) and delete Subprefixes with different IXP names. else: # Update prefix with the new IXP names Stree[prefix] = assign_tuple Sub[prefix] = assign_tuple # Delete subprefix Sub.pop(subnet) final_subnet2country.pop(subnet) else: Sub.pop(subnet) final_subnet2country.pop(subnet) return Stree, Sub, help_tree
def pch_handle_sub(self, reserved_tree, long_mem, IXP_region): ''' Extracts the IXP keys-to-Subnets from the ixp_subnets.csv file. Input: a) filename: The ixp_subnets.csv file. b) self.homepath: The directory path of the database. c) reserved_tree: The SubnetTree containing the reserved Subnets. d) long_mem: Dictionary containing id to long names. e) IXP_region: Dictionary containing {IXP Subnet} = [IXP country, IXP city]. Output: a) subnets: A dictionary with {IXP Subnet}=[IXP long name, IXP short name]. b) IXP_cc: A dictionary with {IXP Subnet}=[IXP country,IXP region]. c) Subnet_names: SubnetTree with Subnet to [IXP long name, IXP short name]. ''' handled_string = string_handler.string_handler() doc = self.file_opener(self.filename_subset, 1) IXP_cc = {} subnets = {} Subnet_names = SubnetTree.SubnetTree() flag = True for line in doc: if flag: flag = False continue temp_string = line.split(',') if len(temp_string) > 5: mykey = temp_string[0] myip = temp_string[6] myip = handled_string.extract_ip(myip, 'Subnet') for ips in myip: ips = handled_string.clean_ip(ips, 'Subnet') if ips != '' and handled_string.string_comparison(temp_string[2], 'Active'): if handled_string.is_valid_ip_address(ips, 'Subnet') and ips not in subnets.keys(): if mykey in long_mem.keys(): IXP_cc[ips] = IXP_region[mykey] [long_name, short_name] = handled_string.clean_long_short( long_mem[mykey], temp_string[1]) if len(long_name) > len(short_name): subnets[ips] = [[long_name, short_name]] else: subnets[ips] = [[short_name, long_name]] elif temp_string[1] != '': [long_name, short_name] = handled_string.clean_long_short("", temp_string[ 1]) if len(short_name) < len(long_name): subnets[ips] = [['', short_name]] else: subnets[ips] = [['', long_name]] else: continue elif ips in subnets.keys(): IXP_cc[ips] = IXP_region[mykey] [long_name, short_name] = handled_string.clean_long_short( long_mem[mykey], temp_string[1]) if short_name > long_name: tmp_name_string = long_name long_name = short_name short_name = tmp_name_string assigned_tuple = [] for IXP in subnets[ips]: assigned_tuple = assigned_tuple + \ handled_string.assign_names( IXP[0], short_name, IXP[1], long_name) subnets[ips] = assigned_tuple if ips in subnets: Subnet_names[ips] = subnets[ips] return (subnets, IXP_cc, Subnet_names)
def extract_additional_info(self, mypath): ''' Input: a) mypath: the directory path of the additional file. ''' mypath += '/configuration/' + self.user_ixps_filename handles = string_handler.string_handler() # Creates the user IXP file if it does not exist. if not os.path.exists(mypath): try: f = open(mypath, 'a') f.close() except: print('Could not create ' + mypath + '.exiting.') sys.exit(0) else: with open(mypath, 'r') as f: # Parses the additional_info.txt file. i = 0 for line in f: i += 1 # Ignores lines with comments. if line.startswith('#'): continue line_split = [line.strip() for line in line.split(', ')] IXP = handles.extract_ip(line, 'Subnet') IXP = IXP[0] if len(IXP) else '' # Imports only IXP Subnets with valid format. if handles.is_valid_ip_address(IXP, 'Subnet', 'Additional') and IXP == line_split[0]: if len(line_split) == 5: ixp_short_name = line_split[1] ixp_full_name = line_split[2] country = line_split[3] city = line_split[4] if IXP not in self.Subnet.keys(): self.additional_info_tree[IXP] = [ixp_full_name, ixp_short_name] self.additional_info_help_tree[IXP] = IXP self.Subnet[IXP] = [ixp_full_name, ixp_short_name] self.pfx2cc[IXP] = [country, city] else: print( 'additional_info.txt: Multiple similar IXP Prefixes detected. Exiting.') sys.exit(0) else: print('Invalid syntax in line ' + str(i) + '. Exiting.') sys.exit(0) # Imports only IXP IPs with valid format. else: IXP = handles.extract_ip(line, 'IP') IXP = IXP[0] if len(IXP) else '' if len(line_split) == 6: try: int(line_split[1]) except: print('additional_info.txt: Invalid syntax in line ' + str(i) + '. Exiting.') sys.exit(0) asn = line_split[1] ixp_short_name = line_split[2] ixp_full_name = line_split[3] country = line_split[4] city = line_split[5] if handles.is_valid_ip_address(IXP, 'IP', 'Additional'): if IXP not in self.ixp_ip2asn: self.ixp_ip2asn[IXP] = [asn] self.Subnet[IXP + '/32'] = [ixp_short_name, ixp_full_name] self.pfx2cc[IXP + '/32'] = [country, city] else: print( 'additional_info.txt: Multiple similar IXP IPs detected. Exiting.') sys.exit(0) else: print( 'additional_info.txt: Invalid IP in line ' + str(i) + '. Exiting.') sys.exit(0) else: print( 'additional_info.txt: Invalid syntax in line ' + str(i) + '. Exiting.') sys.exit(0)
def Subnet_tree(self, Sub, additional_tree, reserved_sub_tree, final_subnet2country): ''' Returns a Subnet Tree containing all the IXP subnets. Input: a) Sub: A dictionary with {Subnet}=[IXP long name,IXP short name]. b) additional_tree: a Subnet Tree containing all the user {IXP Subnet}=IXP Subnet. c) reserved_sub_tree: A SubnetTree containing the reserved Subnets. d) final_subnet2country: A dictionary with {Subnet}=[country,city]. Output: a) Stree: A Subnet Tree with the IXP Subnets. b) Sub: A dictionary with {Subnet}=[IXP long name,IXP short name]. c) help_tree: a Tree with IXP Subnets. ''' Stree = SubnetTree.SubnetTree() help_tree = SubnetTree.SubnetTree() handle_string = string_handler.string_handler() # subnet_keys: A sorted list of prefixes based on subnet mask. subnet_keys = [] for i in Sub.keys(): subnet_keys.append(i.split('/')) subnet_keys = sorted(subnet_keys, key=lambda x: x[1]) for entry in subnet_keys: subnet = entry[0] + '/' + entry[1] check = handle_string.sub_prefix_check(subnet, help_tree) # If a subprefix does not exist and it is not reserved prefix. if not check and not handle_string.sub_prefix_check( subnet, additional_tree) and not handle_string.sub_prefix_check( subnet, reserved_sub_tree): Stree[subnet] = Sub[subnet] help_tree[subnet] = subnet # If a subprefix exist. elif check: assign_tuple = [] for IXP1 in Stree[subnet]: for IXP2 in Sub[subnet]: assign_tuple = assign_tuple + \ handle_string.assign_names( IXP1[1], IXP2[1], IXP1[0], IXP2[0]) # Deletes similar IXP names assigned to the same prefixes. deleted = [] for i in range(0, len(assign_tuple) - 1): for j in range(i + 1, len(assign_tuple)): if len( handle_string.assign_names( assign_tuple[i][0], assign_tuple[j][0], assign_tuple[i][1], assign_tuple[j][1])) == 1: if j not in deleted: deleted = [j] + deleted for node in deleted: del assign_tuple[node] if (assign_tuple == Stree[subnet]): Sub.pop(subnet) final_subnet2country.pop(subnet) else: Stree[subnet] = assign_tuple Sub[subnet] = assign_tuple help_tree[subnet] = subnet else: Sub.pop(subnet) final_subnet2country.pop(subnet) return Stree, Sub, help_tree
def extract_additional_info(self, mypath): ''' Input: a) filename: the additional_info.txt file. ''' mypath += '/configuration/' + self.user_ixps_filename handles = string_handler.string_handler() # Creates the user IXP file if it does not exist. if not os.path.exists(mypath): try: f = open(mypath, 'a') f.close() except: print('Could not create ' + mypath + '.exiting.') sys.exit(0) else: try: f = open(mypath, 'r') except: print('Could not open ' + mypath + '. exiting.') sys.exit(0) # Parses the additional_info.txt file. i = 0 for line in f: i += 1 # Clears the comments. line = line.split('#')[0] line = line.replace(' ', '') line = line.strip() if line == '': continue line_split = line.split(',') IXP = handles.extract_ip(line, 'Subnet') if len(IXP) > 0: IXP = IXP[0] else: IXP = '' # Imports only IXP Subnets with valid format. if handles.is_valid_ip_address( IXP, 'Subnet') and IXP == line_split[0]: if len(line_split) == 5: ixp_full_name = line_split[1] ixp_short_name = line_split[2] city = line_split[3] country = line_split[4] if IXP not in self.Subnet.keys(): self.additional_info_tree[IXP] = [ ixp_full_name, ixp_short_name ] self.additional_info_help_tree[IXP] = IXP self.Subnet[IXP] = [ixp_full_name, ixp_short_name] self.pfx2cc[IXP] = [country, city] else: print( 'additional_info.txt: Multiple similar IXP Prefixes detected. Exiting.' ) sys.exit(0) # # Imports only IXP IPs with valid format. else: print('Invalid syntax in line ' + str(i + 1) + '. Exiting.') sys.exit(0) else: IXP = handles.extract_ip(line, 'IP') if len(IXP) > 0: IXP = IXP[0] else: IXP = '' if len(line_split) == 6: try: int(line_split[1]) except: print( 'additional_info.txt: Invalid syntax in line ' + str(i + 1) + '. Exiting.') sys.exit(0) ASn = line_split[1] ixp_full_name = line_split[2] ixp_short_name = line_split[3] city = line_split[4] country = line_split[5] if handles.is_valid_ip_address( IXP, 'IP') and IXP == line_split[0]: if IXP not in self.IXP_dict.keys(): self.IXP_dict[IXP] = [ASn] self.Subnet[IXP + '/32'] = [ ixp_short_name, ixp_full_name ] self.pfx2cc[IXP + '/32'] = [country, city] else: print( 'additional_info.txt: Multiple similar IXP IPs detected. Exiting.' ) sys.exit(0) else: print( 'additional_info.txt: Invalid syntax in line ' + str(i + 1) + '. Exiting.') sys.exit(0) else: print('additional_info.txt: Invalid syntax in line ' + str(i + 1) + '. Exiting.') sys.exit(0) f.close()
def check_rules(self, path, rule, path_asn, path_cur, ixp_long, ixp_short, asn2names, encounter_type): ''' Checks if the condition part of a rule is satisfied. Input: a) rule: The condition part of the candidate IXP detection rule. b) path_asn: The AS path. c) path_cur: The current hop in the path. d) ixp_long, ixp_short: The long and short IXP names. e) asn2names: A dictionary with a list of lists of short and long IXP names in which an AS is member - {ASN}=[[name long,name short],[name long,name short]...]. f) encounter_type: The resolved IP path based on the encountered types (IXP IP, IXP Prefix, Normal IP, Illegal IP). g) path: The IP path. Output: True if the expression is satisfied, False otherwise. ''' if len(rule) > len(path_asn): return False for i in range(0, len(rule)): if len(path_asn) > path_cur + i - 1: if ('IXP_IP' in rule[i] and '!AS_M' in rule[i]) and 'IXP prefix' not in encounter_type[ path_cur + i - 1] and 'IXP IP' not in encounter_type[path_cur + i - 1]: # if 'IXP_IP' in rule[i] and '!AS_M' in rule[i] and 'IXP # prefix' not in encounter_type[path_cur+i-1]: return False elif 'IXP_IP' in rule[i] and 'AS_M' in rule[ i] and '!' not in rule[ i] and 'IXP IP' not in encounter_type[path_cur + i - 1]: return False elif ('IXP_IP' not in rule[i] or '!AS_M' not in rule[i] ) and 'IXP prefix' in encounter_type[path_cur + i - 1]: return False elif ('IXP_IP' not in rule[i] or 'AS_M' not in rule[i] ) and 'IXP IP' in encounter_type[path_cur + i - 1]: return False # Applies each condition of the condition part of the candidate rule # onto the path. string_h = string_handler.string_handler() check = 0 for i in range(0, len(rule)): # The current condition of the condition part of the rule. current = path_cur + i - 1 expression = rule[i] # Checking for IXP membership based on a non-IXP IP. if '!AS_M' in expression and 'and' not in expression and path_cur != current: # Finds the path_asn in the routeview path_asn dict. If not, an # assessment is not possible. check = check + 1 if path_asn[current] == '*' and encounter_type[ current] != 'IXP prefix': return False if encounter_type[path_cur] == 'IXP IP' or encounter_type[ path_cur] == 'IXP prefix': as_names = '' if path_asn[current] in asn2names.keys(): as_names = asn2names[path_asn[current]] ix_long = ixp_long[path_cur] ix_short = ixp_short[path_cur] else: as_names = '' if path_asn[path_cur] in asn2names.keys(): as_names = asn2names[path_asn[path_cur]] ix_long = ixp_long[current] ix_short = ixp_short[current] for node in as_names: for name in node: if (string_h.string_comparison(ix_long, name) or string_h.string_comparison(ix_short, name)): return False if not self.check_number(rule, expression, path_asn, current, i, encounter_type, '!AS_M'): return False elif 'AS_M' in expression and 'and' not in expression and path_cur != current: if path_asn[current] == '*' and encounter_type[ current] != 'IXP prefix': return False check = check + 1 flag = False if encounter_type[path_cur] == 'IXP IP' or encounter_type[ path_cur] == 'IXP prefix': as_names = '' if path_asn[current] in asn2names.keys(): as_names = asn2names[path_asn[current]] ix_long = ixp_long[path_cur] ix_short = ixp_short[path_cur] else: as_names = '' if path_asn[path_cur] in asn2names.keys(): as_names = asn2names[path_asn[path_cur]] ix_long = ixp_long[current] ix_short = ixp_short[current] for node in (as_names): for name in node: if string_h.string_comparison( ix_long, name) or string_h.string_comparison( ix_short, name): flag = 1 break if not flag: return False if not self.check_number(rule, expression, path_asn, current, i, encounter_type, 'AS_M'): return False if '*' in expression and path[current] != '*': return False # Checking for IXP IP or Prefix based on either IXP membership or # Prefixes data. if 'IXP_IP' in expression and '!AS_M' in expression: check = check + 1 if not self.check_names(rule, expression, current, i, encounter_type, 'IXP_IP', ixp_long, ixp_short): return False elif not self.check_number(rule, expression, path_asn, current, i, encounter_type, '!AS_M'): return False elif 'IXP_IP' in expression and 'AS_M' in expression: check = check + 1 if not self.check_names(rule, expression, current, i, encounter_type, 'IXP_IP', ixp_long, ixp_short): return False elif not self.check_number(rule, expression, path_asn, current, i, encounter_type, 'AS_M'): return False if len(rule) > 2 and len(path_asn) > current + 1: check = check + 1 if not self.check_edges(rule, path_asn, current, 'AS_M', ixp_long, ixp_short): return False elif not self.check_edges(rule, path_asn, current, 'IXP_IP', ixp_long, ixp_short): return False if check: return True else: return False