def analyze_date_caida(self): date_dir = self.data_dir + "/" + self.date if (not os.path.exists(date_dir) or not os.listdir(date_dir)): print("path is empty or not existent") exit() for fn in os.listdir(date_dir): if (re.findall(".*\.gz$", fn)): print "analyzing " + fn + " ..." if (os.path.exists(date_dir + "/" + fn.rstrip(".gz"))): os.system("rm -f " + date_dir + "/" + fn.rstrip(".gz")) warts_name = date_dir + "/" + fn.strip(".gz") dump_name = date_dir + "/" + warts_name.split('.')[-2] monitor_name = warts_name.split('.')[-2] temp_topo = ip_topo.topo_graph() temp_topo.build_popen_pipeline(date_dir, fn) self.topo.merge(temp_topo, monitor_name) print fn + " analyzed" print self.topo.get_node_num() print self.topo.get_target_num() print "exporting topo to db" self.export_topo_to_db() print "finished exporting topo to db"
def __init__(self, date, file_name=""): self.config = config.get_config_section_dict("config.ini", "data") self.data_dir = self.config["data_dir"] self.date = date self.topo = ip_topo.topo_graph() self.db_helper = db_helper.db_helper()
def analyze_date_caida(self): date_dir = self.data_dir + "/" + self.date if (not os.path.exists(date_dir) or not os.listdir(date_dir)): print("path is empty or not existent") exit() for fn in os.listdir(date_dir): if (re.findall(".*\.gz$", fn)): print "analyzing " + fn + " ..." if (os.path.exists(date_dir + "/" + fn.rstrip(".gz"))): os.system("rm -f " + date_dir + "/" + fn.rstrip(".gz")) os.system("gzip -d -k -q " + date_dir + "/" + fn) warts_name = date_dir + "/" + fn.strip(".gz") dump_name = date_dir + "/" + warts_name.split('.')[-2] monitor_name = warts_name.split('.')[-2] os.system("sc_analysis_dump " + warts_name + " > " + dump_name) temp_topo = ip_topo.topo_graph() temp_topo.build(dump_name) self.topo.merge(temp_topo, monitor_name) os.system("rm -f " + warts_name) os.system("rm -f " + dump_name) print fn + " analyzed" print self.topo.get_node_num() print self.topo.get_target_num() #self.topo.build_networkx_graph() #print "pickling topo" #self.pickle_topo() #print "finished pickling topo" print "exporting topo to db" self.export_topo_to_db() print "finished exporting topo to db"
def analyze_date_caida_pipeline(self): self.topo = ip_topo.topo_graph() self.topo.build_pipeline() print self.topo.get_node_num(), self.topo.get_target_num() print "exporting topo to db" self.export_topo_to_db() print "finished exporting topo to db"
file_08 = [] #file_12 = []; file_16 = [] print("building topo_08") url_list = caida.get_time_list("caida", "20080617") dir = "data/caida/ip/20080617/all/" for u in url_list: time = u.split('/')[9].split('.')[4] node = u.split('/')[9].split('.')[5] if (time == "20080617"): raw_file = "caida.ip." + time + "." + node file_08.append(raw_file) topo_08 = ip_topo.topo_graph(ip_topo.get_src(dir + file_08[0]), False) topo_08.build(dir + file_08[0], "caida", False, False) for i in range(1, len(file_08)): topo = ip_topo.topo_graph(ip_topo.get_src(dir + file_08[i]), False) topo.build(dir + file_08[i], "caida", False, False) topo_08.merge(topo) print str(i + 1) + " of " + str(len(file_08)) + " done." ''' print("building topo_12"); url_list = caida.get_time_list("caida","20120617"); dir = "data/caida/ip/20120617/all/"; for u in url_list: time = u.split('/')[9].split('.')[4]; node = u.split('/')[9].split('.')[5]; if(time=="20120617"): raw_file = "caida.ip."+time+"."+node;
import ip_topo topo = ip_topo.topo_graph(ip_topo.get_src_lg(),False); topo.build("traces.txt","lg",False,False); topo.calc_deg(); topo.calc_knn(); topo.draw_deg("lg"); topo.draw_knn("lg"); topo.draw_path("lg");
file_08 = []; #file_12 = []; file_16 = []; print("building topo_08"); url_list = caida.get_time_list("caida","20080617"); dir = "data/caida/ip/20080617/all/"; for u in url_list: time = u.split('/')[9].split('.')[4]; node = u.split('/')[9].split('.')[5]; if(time=="20080617"): raw_file = "caida.ip."+time+"."+node; file_08.append(raw_file); topo_08 = ip_topo.topo_graph(ip_topo.get_src(dir+file_08[0]),False); topo_08.build(dir+file_08[0],"caida",False,False); for i in range(1, len(file_08)): topo = ip_topo.topo_graph(ip_topo.get_src(dir+file_08[i]),False); topo.build(dir+file_08[i],"caida",False,False); topo_08.merge(topo); print str(i+1)+" of "+str(len(file_08))+" done."; ''' print("building topo_12"); url_list = caida.get_time_list("caida","20120617"); dir = "data/caida/ip/20120617/all/"; for u in url_list: time = u.split('/')[9].split('.')[4]; node = u.split('/')[9].split('.')[5]; if(time=="20120617"):
def main(argv): if (len(argv) != 6 and len(argv) != 7): usage(); exit(); source = argv[1]; type = argv[2]; time = argv[3]; node = argv[4]; target = argv[5]; is_parse = ""; if (len(argv) == 7): is_parse = argv[6]; if (source == "caida" and type == "ip" and node != "all"): dir = "data/caida/ip/"+time+"/"+node+"/"; file = source+"."+type+"."+time+"."+node+"."+target; raw_file = source+"."+type+"."+time+"."+node; if not os.path.exists(dir): os.makedirs(dir); if os.path.exists(dir+file): print "already exists"; exit(); elif not os.path.exists(dir+raw_file): print "raw file does not exist, start downloading..."; auth = read_auth("caida"); if ( len(auth) != 2 ): print "auth failed"; exit(); url = caida.get_url("caida", time, node); if ( url == None): print "no such record found"; exit(); download_worker.download_caida_restricted_worker(url, dir, raw_file+".warts.gz", auth[0], auth[1]); print "finished downloading."; print "dumping..."; os.system("gunzip -q "+dir+raw_file+".warts.gz"); os.system("sc_analysis_dump "+dir+raw_file+".warts > "+dir+raw_file); print "finished dumping."; if not is_parse == "not": print "building topo..."; topo = ip_topo.topo_graph(ip_topo.get_src(dir+raw_file),True); topo.build(dir+raw_file,"caida",True,True); topo.disp_stats(); print "generating map..."; topo.generate_map(); print "exporting map..."; topo.export_map(dir+raw_file); print "exporting simplified topo..."; topo.export_topo_simplified(dir+raw_file); print "exporting graphviz..."; topo.export_graphviz(dir+raw_file); print "exporting degree..."; topo.export_degree(dir+raw_file); print "exporting path tree..."; topo.export_path_tree(dir+raw_file); print "exporting border..."; topo.export_border_ip(dir+raw_file); elif (source == "caida" and type == "ip" and node == "all"): dir = "data/caida/ip/"+time+"/all/"; file = source+"."+type+"."+time+".all."+target; if not os.path.exists(dir): os.makedirs(dir); if os.path.exists(dir+file): print "already exists"; exit(); else: url_list = caida.get_time_list("caida", time); for u in url_list: node = u.split('/')[9].split('.')[5]; raw_file = source+"."+type+"."+time+"."+node; if not os.path.exists(dir+raw_file): print "raw file "+raw_file+" does not exist, start downloading..."; auth = read_auth("caida"); if ( len(auth) != 2 ): print "auth failed"; exit(); url = caida.get_url("caida", time, node); if ( url == None): print "no such record found"; download_worker.download_caida_restricted_worker(url, dir, raw_file+".warts.gz", auth[0], auth[1]); print "finished downloading."; print "dumping..."; os.system("gunzip -q "+dir+raw_file+".warts.gz"); os.system("sc_analysis_dump "+dir+raw_file+".warts > "+dir+raw_file); print "finished dumping."; elif (source == "iplane" and type == "ip"): dir = "data/iplane/ip/"+time+"/"; file = source+"."+type+"."+time+"."+node+"."+target; raw_file = source+"."+type+"."+time; url = iplane.get_url("iplane", time); dir_out = url.split('/')[6].split('.')[0]; if not os.path.exists(dir): os.makedirs(dir); if os.path.exists(dir+file): print "already exists"; exit(); if not os.path.exists(dir+raw_file+".tar.gz"): print "raw file does not exist, start downloading..."; auth = read_auth("iplane"); if ( len(auth) != 2 ): print "auth failed"; exit(); if ( url == None ): print "no such record found"; exit(); download_worker.download_iplane_restricted_worker(url, dir, raw_file+".tar.gz", auth[0], auth[1]); print "finished downloading."; if not os.path.exists(dir+dir_out): print "uncompressing ..."; os.system("tar -zxvf "+dir+raw_file+".tar.gz"+" -C "+dir); if not os.path.exists(dir+raw_file+".ls"): os.system("ls "+dir+dir_out+" > "+dir+raw_file+".ls"); f_ls = open(dir+raw_file+".ls"); for line in f_ls.readlines(): n = line.split('.',2)[2].strip('\n'); out_file = raw_file+"."+n os.system("./readoutfile "+dir+dir_out+"/"+line.strip('\n')+" > "+dir+out_file); print ("./readoutfile "+dir+dir_out+"/"+line.strip('\n')+" > "+dir+out_file); f_ls.close(); print "finished dumping."; is_included = False; f_ls = open(dir+raw_file+".ls"); for line in f_ls.readlines(): n = line.split('.',2)[2].strip('\n'); if (n == node): is_included = True; f_ls.close(); if(not is_included): print "no such record"; exit(); print "building topo..."; topo = ip_topo.topo_graph(ip_topo.get_src_iplane(dir+raw_file+"."+node),True); topo.build(dir+raw_file+"."+node,"iplane",True,True); topo.disp_stats(); print "generating map..."; topo.generate_map(); print "exporting map..."; topo.export_map(dir+raw_file+"."+node); print "exporting simplified topo..."; topo.export_topo_simplified(dir+raw_file+"."+node); print "exporting graphviz..."; topo.export_graphviz(dir+raw_file+"."+node); print "exporting degree..."; topo.export_degree(dir+raw_file+"."+node); print "exporting path tree..."; topo.export_path_tree(dir+raw_file+"."+node); print "exporting border..."; topo.export_border_ip(dir+raw_file);