def print_paths_to_database(paths, reverse_map, table_name): # Timeout = 6000s insert_string = "INSERT INTO %s VALUES (?, ?, ?, ?, ?, ?, ?)" % table_name queries = [] for p_node in paths: path_string = "" for port in p_node["visits"]: path_string += ("%d " % port) path_string += ("%d " % p_node["port"]) port_count = len(p_node["visits"]) + 1 rl_id = "" for (n, r, s) in p_node["hdr"].applied_rule_ids: rl_id += (r + " ") rule_count = len(p_node["hdr"].applied_rule_ids) input_port = p_node["visits"][0] output_port = p_node["port"] output_hs = p_node["hdr"].copy() applied_rule_ids = list(output_hs.applied_rule_ids) input_hs = trace_hs_back(applied_rule_ids, output_hs, output_port)[0] header_string = json.dumps(parse_hs(juniperRouter(1).hs_format, input_hs.hs_list[0])) #header_string = byte_array_to_pretty_hs_string(input_hs.hs_list[0]) queries.append((header_string, input_port, output_port, path_string, port_count, rl_id, rule_count)) conn = sqlite3.connect(DATABASE_FILE, 6000) for query in queries: conn.execute(insert_string, query) conn.commit() conn.close()
def load_internet2_backbone_port_to_id_map(): f = open("Internet2/port_map.txt",'r') id_to_name = {} map = {} rtr = "" cs = juniperRouter(1) for line in f: if line.startswith("$"): rtr = line[1:].strip() map[rtr] = {} elif line != "": tokens = line.strip().split(":") map[rtr][tokens[0]] = int(tokens[-1]) id_to_name[tokens[-1]] = "%s-%s"%(rtr,":".join(tokens[0:-1])) out_port = int(tokens[-1]) + cs.PORT_TYPE_MULTIPLIER * cs.OUTPUT_PORT_TYPE_CONST id_to_name["%s"%out_port] = "%s-%s"%(rtr,":".join(tokens[0:-1])) return (map,id_to_name)
def load_internet2_backbone_port_to_id_map(): f = open("work/Internet2/port_map.txt", 'r') id_to_name = {} map = {} rtr = "" cs = juniperRouter(1) for line in f: if line.startswith("$"): rtr = line[1:].strip() map[rtr] = {} elif line != "": tokens = line.strip().split(":") map[rtr][tokens[0]] = int(tokens[-1]) id_to_name[tokens[-1]] = "%s-%s" % (rtr, ":".join(tokens[0:-1])) out_port = int( tokens[-1] ) + cs.PORT_TYPE_MULTIPLIER * cs.OUTPUT_PORT_TYPE_CONST id_to_name["%s" % out_port] = "%s-%s" % (rtr, ":".join(tokens[0:-1])) return (map, id_to_name)
def main(): global ntf_global global ttf_global global dst_port_ids_global ntf_global = load_internet2_backbone_ntf() ttf_global = load_internet2_backbone_ttf() (port_map, port_reverse_map) = load_internet2_backbone_port_to_id_map() cs = juniperRouter(1) output_port_addition = cs.PORT_TYPE_MULTIPLIER * cs.OUTPUT_PORT_TYPE_CONST all_x = byte_array_get_all_x(ntf_global.length) #cs.set_field(all_x, "vlan", 32, 0) #cs.set_field(all_x, "ip_dst", dotted_ip_to_int("64.57.28.243"), 8) #cs.set_field(all_x, "ip_src", dotted_ip_to_int("172.27.76.233"), 0) #cs.set_field(all_x, "tcp_dst", 22, 0) #cs.set_field(all_x, "ip_proto", 6, 0) test_pkt = headerspace(ntf_global.length) test_pkt.add_hs(all_x) src_port_id = port_map["atla"]["xe-0/1/1"] dst_port_ids_global = [port_map["atla"]["xe-1/0/2"] + output_port_addition] st = time() paths = find_reachability_multiprocess(src_port_id, test_pkt) #paths = find_reachability(ntf_global, ttf_global, src_port_id, dst_port_ids_global, test_pkt) en = time() print_loops(paths, port_reverse_map) print len(paths) #loops = detect_loop(ntf,ttf,loop_port_ids,port_reverse_map,None,output_port_addition) #en = time() #print_loops(loops, port_reverse_map) #print len(loops) print en - st
def main(): global ntf_global global ttf_global global dst_port_ids_global ntf_global = load_internet2_backbone_ntf() ttf_global = load_internet2_backbone_ttf() (port_map,port_reverse_map) = load_internet2_backbone_port_to_id_map() cs = juniperRouter(1) output_port_addition = cs.PORT_TYPE_MULTIPLIER * cs.OUTPUT_PORT_TYPE_CONST all_x = byte_array_get_all_x(ntf_global.length) #cs.set_field(all_x, "vlan", 32, 0) #cs.set_field(all_x, "ip_dst", dotted_ip_to_int("64.57.28.243"), 8) #cs.set_field(all_x, "ip_src", dotted_ip_to_int("172.27.76.233"), 0) #cs.set_field(all_x, "tcp_dst", 22, 0) #cs.set_field(all_x, "ip_proto", 6, 0) test_pkt = headerspace(ntf_global.length) test_pkt.add_hs(all_x) src_port_id = port_map["atla"]["xe-0/1/1"] dst_port_ids_global = [port_map["atla"]["xe-1/0/2"]+output_port_addition] st = time() paths = find_reachability_multiprocess(src_port_id,test_pkt) #paths = find_reachability(ntf_global, ttf_global, src_port_id, dst_port_ids_global, test_pkt) en = time() print_loops(paths, port_reverse_map) print len(paths) #loops = detect_loop(ntf,ttf,loop_port_ids,port_reverse_map,None,output_port_addition) #en = time() #print_loops(loops, port_reverse_map) #print len(loops) print en-st
def main(): global src_port_ids_global global dst_port_ids_global global port_map_global global port_reverse_map_global global ntf_global global ttf_global global DATABASE_FILE parser = ArgumentParser(description="Generate Test Packets for Internet2") parser.add_argument("-p", dest="percentage", type=int, default="100", help="Percentage of test terminals") parser.add_argument("-f", dest="filename", default="internet2.sqlite", help="Filename of the database") parser.add_argument("-e", action="store_true", default=False, help="Edge port only") args = parser.parse_args() DATABASE_FILE = "work/%s" % args.filename cs = juniperRouter(1) output_port_addition = cs.PORT_TYPE_MULTIPLIER * cs.OUTPUT_PORT_TYPE_CONST # Load .tf files ntf_global = load_internet2_backbone_ntf() ttf_global = load_internet2_backbone_ttf() (port_map_global, port_reverse_map_global) = load_internet2_backbone_port_to_id_map() # Initialize the database if os.access(DATABASE_FILE, os.F_OK): os.remove(DATABASE_FILE) conn = sqlite3.connect(DATABASE_FILE) conn.execute( 'CREATE TABLE %s (rule TEXT, input_port TEXT, output_port TEXT, action TEXT, file TEXT, line TEXT)' % TABLE_NETWORK_RULES) conn.execute( 'CREATE TABLE %s (rule TEXT, input_port TEXT, output_port TEXT)' % TABLE_TOPOLOGY_RULES) conn.execute( 'CREATE TABLE %s (header TEXT, input_port INTEGER, output_port INTEGER, ports TEXT, no_of_ports INTEGER, rules TEXT, no_of_rules INTEGER)' % TABLE_TEST_PACKETS) conn.execute( 'CREATE TABLE %s (header TEXT, input_port INTEGER, output_port INTEGER, ports TEXT, no_of_ports INTEGER, rules TEXT, no_of_rules INTEGER)' % TABLE_TEST_PACKETS_LOCALLY_COMPRESSED) conn.execute('CREATE TABLE %s (rules TEXT, no_of_rules INTEGER)' % TABLE_TEST_PACKETS_GLOBALLY_COMPRESSED) conn.execute('CREATE TABLE %s (rule TEXT)' % TABLE_RESULT_RULES) rule_count = 0 for tf in ntf_global.tf_list: rule_count += len(tf.rules) for rule in tf.rules: query = "INSERT INTO %s VALUES (?, ?, ?, ?, ?, ?)" % TABLE_NETWORK_RULES conn.execute( query, (rule['id'], ' '.join(map(str, rule['in_ports'])), ' '.join( map(str, rule['out_ports'])), rule['action'], rule["file"], ' '.join(map(str, rule["line"])))) print "Total Rules: %d" % rule_count conn.commit() rule_count = len(ttf_global.rules) for rule in ttf_global.rules: query = "INSERT INTO %s VALUES (?, ?, ?)" % TABLE_TOPOLOGY_RULES conn.execute(query, (rule['id'], ' '.join(map( str, rule['in_ports'])), ' '.join(map(str, rule['out_ports'])))) print "Total Links: %d" % rule_count # Generate all ports for rtr in port_map_global.keys(): src_port_ids_global |= set(port_map_global[rtr].values()) total_length = len(src_port_ids_global) if args.e == True: for rule in ttf_global.rules: if rule['out_ports'][0] in src_port_ids_global: src_port_ids_global.remove(rule['out_ports'][0]) new_length = len(src_port_ids_global) * args.percentage / 100 src_port_ids_global = random.sample(src_port_ids_global, new_length) print "Total Length: %d" % total_length print "New Length: %d" % new_length for port in src_port_ids_global: port += output_port_addition dst_port_ids_global.add(port) #src_port_ids_global = [300013] #dst_port_ids_global = [320010] conn.commit() conn.close() # Run reachability start_time = time.time() pool = Pool() result = pool.map_async(find_test_packets, src_port_ids_global) # Close pool.close() pool.join() end_time = time.time() test_packet_count = result.get() total_paths = sum(test_packet_count) print "========== Before Compression =========" print "Total Paths = %d" % total_paths print "Average packets per port = %f" % (float(total_paths) / len(src_port_ids_global)) print "Total Time = %fs" % (end_time - start_time) #Global Compressing start_time = time.time() conn = sqlite3.connect(DATABASE_FILE, 6000) result_rule_lists = [] query = "SELECT rules FROM %s" % TABLE_TEST_PACKETS_LOCALLY_COMPRESSED rows = conn.execute(query) for row in rows: result_rule_lists.append(row[0].split()) conn.close() chunk_size = 80000 while (True): print "Start a new round!" conn = sqlite3.connect(DATABASE_FILE, 6000) conn.execute('DROP TABLE IF EXISTS %s' % TABLE_SCRATCHPAD) conn.execute('CREATE TABLE %s (rules TEXT, no_of_rules INTEGER)' % TABLE_SCRATCHPAD) conn.commit() conn.close() start_len = len(result_rule_lists) print start_len pool = Pool() no_of_chunks = len(result_rule_lists) / chunk_size + 1 rule_list_chunks = chunks(result_rule_lists, no_of_chunks) result = pool.map_async(rule_lists_compress, rule_list_chunks) # Close pool.close() pool.join() result.get() print "End of this round." result_rule_lists = read_rule_lists_from_database(TABLE_SCRATCHPAD) end_len = len(result_rule_lists) if (float(end_len) / float(start_len) > 0.99): break end_time = time.time() query = "INSERT INTO %s VALUES (?, ?)" % TABLE_TEST_PACKETS_GLOBALLY_COMPRESSED query2 = "INSERT INTO %s VALUES (?)" % TABLE_RESULT_RULES total_paths = len(result_rule_lists) total_length = 0 conn = sqlite3.connect(DATABASE_FILE, 6000) conn.execute('DROP TABLE IF EXISTS %s' % TABLE_TEST_PACKETS_GLOBALLY_COMPRESSED) conn.execute('CREATE TABLE %s (rules TEXT, no_of_rules INTEGER)' % TABLE_TEST_PACKETS_GLOBALLY_COMPRESSED) for rule_list in result_rule_lists: total_length += len(rule_list) conn.execute(query, (" ".join(rule_list), len(rule_list))) for rule in rule_list: conn.execute(query2, (rule, )) conn.commit() conn.close() print "========== After Compression =========" print "Total Paths = %d" % total_paths print "Average packets per port = %f" % (float(total_paths) / len(src_port_ids_global)) print "Average length of rule list = %f" % (float(total_length) / total_paths) print "Total Time = %fs" % (end_time - start_time)
def main(): global src_port_ids_global global dst_port_ids_global global port_map_global global port_reverse_map_global global ntf_global global ttf_global global DATABASE_FILE parser = ArgumentParser(description="Generate Test Packets for Internet2") parser.add_argument("-p", dest="percentage", type=int, default="100", help="Percentage of test terminals") parser.add_argument("-f", dest="filename", default="internet2.sqlite", help="Filename of the database") parser.add_argument("-e", action="store_true", default=False, help="Edge port only") args = parser.parse_args() DATABASE_FILE = "work/%s" % args.filename cs = juniperRouter(1) output_port_addition = cs.PORT_TYPE_MULTIPLIER * cs.OUTPUT_PORT_TYPE_CONST # Load .tf files ntf_global = load_internet2_backbone_ntf() ttf_global = load_internet2_backbone_ttf() (port_map_global, port_reverse_map_global) = load_internet2_backbone_port_to_id_map() # Initialize the database if os.access(DATABASE_FILE, os.F_OK): os.remove(DATABASE_FILE) conn = sqlite3.connect(DATABASE_FILE) conn.execute('CREATE TABLE %s (rule TEXT, input_port TEXT, output_port TEXT, action TEXT, file TEXT, line TEXT)' % TABLE_NETWORK_RULES) conn.execute('CREATE TABLE %s (rule TEXT, input_port TEXT, output_port TEXT)' % TABLE_TOPOLOGY_RULES) conn.execute('CREATE TABLE %s (header TEXT, input_port INTEGER, output_port INTEGER, ports TEXT, no_of_ports INTEGER, rules TEXT, no_of_rules INTEGER)' % TABLE_TEST_PACKETS) conn.execute('CREATE TABLE %s (header TEXT, input_port INTEGER, output_port INTEGER, ports TEXT, no_of_ports INTEGER, rules TEXT, no_of_rules INTEGER)' % TABLE_TEST_PACKETS_LOCALLY_COMPRESSED) conn.execute('CREATE TABLE %s (rules TEXT, no_of_rules INTEGER)' % TABLE_TEST_PACKETS_GLOBALLY_COMPRESSED) conn.execute('CREATE TABLE %s (rule TEXT)' % TABLE_RESULT_RULES) rule_count = 0 for tf in ntf_global.tf_list: rule_count += len(tf.rules) for rule in tf.rules: query = "INSERT INTO %s VALUES (?, ?, ?, ?, ?, ?)" % TABLE_NETWORK_RULES conn.execute(query, (rule['id'],' '.join(map(str, rule['in_ports'])), ' '.join(map(str, rule['out_ports'])), rule['action'], rule["file"], ' '.join(map(str, rule["line"])))) print "Total Rules: %d" % rule_count conn.commit() rule_count = len(ttf_global.rules) for rule in ttf_global.rules: query = "INSERT INTO %s VALUES (?, ?, ?)" % TABLE_TOPOLOGY_RULES conn.execute(query, (rule['id'],' '.join(map(str, rule['in_ports'])), ' '.join(map(str, rule['out_ports'])))) print "Total Links: %d" % rule_count # Generate all ports for rtr in port_map_global.keys(): src_port_ids_global |= set(port_map_global[rtr].values()) total_length = len(src_port_ids_global) if args.e == True: for rule in ttf_global.rules: if rule['out_ports'][0] in src_port_ids_global: src_port_ids_global.remove(rule['out_ports'][0]) new_length = len(src_port_ids_global)* args.percentage / 100 src_port_ids_global = random.sample(src_port_ids_global, new_length) print "Total Length: %d" % total_length print "New Length: %d" % new_length for port in src_port_ids_global: port += output_port_addition dst_port_ids_global.add(port) #src_port_ids_global = [300013] #dst_port_ids_global = [320010] conn.commit() conn.close() # Run reachability start_time = time.time() pool = Pool() result = pool.map_async(find_test_packets, src_port_ids_global) # Close pool.close() pool.join() end_time = time.time() test_packet_count = result.get() total_paths = sum(test_packet_count) print "========== Before Compression =========" print "Total Paths = %d" % total_paths print "Average packets per port = %f" % (float(total_paths) / len(src_port_ids_global)) print "Total Time = %fs" % (end_time - start_time) #Global Compressing start_time = time.time() conn = sqlite3.connect(DATABASE_FILE, 6000) result_rule_lists = [] query = "SELECT rules FROM %s" % TABLE_TEST_PACKETS_LOCALLY_COMPRESSED rows = conn.execute(query) for row in rows: result_rule_lists.append(row[0].split()) conn.close() chunk_size = 80000 while(True): print "Start a new round!" conn = sqlite3.connect(DATABASE_FILE, 6000) conn.execute('DROP TABLE IF EXISTS %s' % TABLE_SCRATCHPAD) conn.execute('CREATE TABLE %s (rules TEXT, no_of_rules INTEGER)' % TABLE_SCRATCHPAD) conn.commit() conn.close() start_len = len(result_rule_lists) print start_len pool = Pool() no_of_chunks = len(result_rule_lists) / chunk_size + 1 rule_list_chunks = chunks(result_rule_lists, no_of_chunks) result = pool.map_async(rule_lists_compress, rule_list_chunks) # Close pool.close() pool.join() result.get() print "End of this round." result_rule_lists = read_rule_lists_from_database(TABLE_SCRATCHPAD) end_len = len(result_rule_lists) if(float(end_len) / float(start_len) > 0.99): break end_time = time.time() query = "INSERT INTO %s VALUES (?, ?)" % TABLE_TEST_PACKETS_GLOBALLY_COMPRESSED query2 = "INSERT INTO %s VALUES (?)" % TABLE_RESULT_RULES total_paths = len(result_rule_lists) total_length = 0 conn = sqlite3.connect(DATABASE_FILE, 6000) conn.execute('DROP TABLE IF EXISTS %s' % TABLE_TEST_PACKETS_GLOBALLY_COMPRESSED) conn.execute('CREATE TABLE %s (rules TEXT, no_of_rules INTEGER)' % TABLE_TEST_PACKETS_GLOBALLY_COMPRESSED) for rule_list in result_rule_lists: total_length += len(rule_list) conn.execute(query, (" ".join(rule_list), len(rule_list))) for rule in rule_list: conn.execute(query2, (rule,)) conn.commit() conn.close() print "========== After Compression =========" print "Total Paths = %d" % total_paths print "Average packets per port = %f" % (float(total_paths) / len(src_port_ids_global)) print "Average length of rule list = %f" % (float(total_length) / total_paths) print "Total Time = %fs" % (end_time - start_time)