Beispiel #1
0
    def analyze_date_caida(self):
        date_dir = self.data_dir + "/" + self.date
        if (not os.path.exists(date_dir) or not os.listdir(date_dir)):
            print("path is empty or not existent")
            exit()

        for fn in os.listdir(date_dir):
            if (re.findall(".*\.gz$", fn)):
                print "analyzing " + fn + " ..."
                if (os.path.exists(date_dir + "/" + fn.rstrip(".gz"))):
                    os.system("rm -f " + date_dir + "/" + fn.rstrip(".gz"))
                warts_name = date_dir + "/" + fn.strip(".gz")
                dump_name = date_dir + "/" + warts_name.split('.')[-2]
                monitor_name = warts_name.split('.')[-2]

                temp_topo = ip_topo.topo_graph()
                temp_topo.build_popen_pipeline(date_dir, fn)

                self.topo.merge(temp_topo, monitor_name)
                print fn + " analyzed"
                print self.topo.get_node_num()
                print self.topo.get_target_num()

        print "exporting topo to db"
        self.export_topo_to_db()
        print "finished exporting topo to db"
Beispiel #2
0
    def __init__(self, date, file_name=""):
        self.config = config.get_config_section_dict("config.ini", "data")
        self.data_dir = self.config["data_dir"]
        self.date = date

        self.topo = ip_topo.topo_graph()
        self.db_helper = db_helper.db_helper()
Beispiel #3
0
    def analyze_date_caida(self):
        date_dir = self.data_dir + "/" + self.date
        if (not os.path.exists(date_dir) or not os.listdir(date_dir)):
            print("path is empty or not existent")
            exit()

        for fn in os.listdir(date_dir):
            if (re.findall(".*\.gz$", fn)):
                print "analyzing " + fn + " ..."
                if (os.path.exists(date_dir + "/" + fn.rstrip(".gz"))):
                    os.system("rm -f " + date_dir + "/" + fn.rstrip(".gz"))
                os.system("gzip -d -k -q " + date_dir + "/" + fn)
                warts_name = date_dir + "/" + fn.strip(".gz")
                dump_name = date_dir + "/" + warts_name.split('.')[-2]
                monitor_name = warts_name.split('.')[-2]
                os.system("sc_analysis_dump " + warts_name + " > " + dump_name)

                temp_topo = ip_topo.topo_graph()
                temp_topo.build(dump_name)
                self.topo.merge(temp_topo, monitor_name)

                os.system("rm -f " + warts_name)
                os.system("rm -f " + dump_name)
                print fn + " analyzed"
                print self.topo.get_node_num()
                print self.topo.get_target_num()

        #self.topo.build_networkx_graph()
        #print "pickling topo"
        #self.pickle_topo()
        #print "finished pickling topo"

        print "exporting topo to db"
        self.export_topo_to_db()
        print "finished exporting topo to db"
Beispiel #4
0
    def analyze_date_caida_pipeline(self):
        self.topo = ip_topo.topo_graph()
        self.topo.build_pipeline()
        print self.topo.get_node_num(), self.topo.get_target_num()

        print "exporting topo to db"
        self.export_topo_to_db()
        print "finished exporting topo to db"
Beispiel #5
0
file_08 = []
#file_12 = [];
file_16 = []

print("building topo_08")
url_list = caida.get_time_list("caida", "20080617")
dir = "data/caida/ip/20080617/all/"
for u in url_list:
    time = u.split('/')[9].split('.')[4]
    node = u.split('/')[9].split('.')[5]
    if (time == "20080617"):
        raw_file = "caida.ip." + time + "." + node
        file_08.append(raw_file)

topo_08 = ip_topo.topo_graph(ip_topo.get_src(dir + file_08[0]), False)
topo_08.build(dir + file_08[0], "caida", False, False)
for i in range(1, len(file_08)):
    topo = ip_topo.topo_graph(ip_topo.get_src(dir + file_08[i]), False)
    topo.build(dir + file_08[i], "caida", False, False)
    topo_08.merge(topo)
    print str(i + 1) + " of " + str(len(file_08)) + " done."
'''
print("building topo_12");
url_list = caida.get_time_list("caida","20120617");
dir = "data/caida/ip/20120617/all/";
for u in url_list:
	time = u.split('/')[9].split('.')[4];
	node = u.split('/')[9].split('.')[5];
	if(time=="20120617"):
		raw_file = "caida.ip."+time+"."+node;
Beispiel #6
0
import ip_topo

topo = ip_topo.topo_graph(ip_topo.get_src_lg(),False);
topo.build("traces.txt","lg",False,False);
topo.calc_deg();
topo.calc_knn();
topo.draw_deg("lg");
topo.draw_knn("lg");
topo.draw_path("lg");
Beispiel #7
0
file_08 = [];
#file_12 = [];
file_16 = [];


print("building topo_08");
url_list = caida.get_time_list("caida","20080617");
dir = "data/caida/ip/20080617/all/";
for u in url_list:
	time = u.split('/')[9].split('.')[4];
	node = u.split('/')[9].split('.')[5];
	if(time=="20080617"):
		raw_file = "caida.ip."+time+"."+node;
		file_08.append(raw_file);

topo_08 = ip_topo.topo_graph(ip_topo.get_src(dir+file_08[0]),False);
topo_08.build(dir+file_08[0],"caida",False,False);
for i in range(1, len(file_08)):
	topo = ip_topo.topo_graph(ip_topo.get_src(dir+file_08[i]),False);
	topo.build(dir+file_08[i],"caida",False,False);
	topo_08.merge(topo);
	print str(i+1)+" of "+str(len(file_08))+" done.";

'''
print("building topo_12");
url_list = caida.get_time_list("caida","20120617");
dir = "data/caida/ip/20120617/all/";
for u in url_list:
	time = u.split('/')[9].split('.')[4];
	node = u.split('/')[9].split('.')[5];
	if(time=="20120617"):
Beispiel #8
0
def main(argv):
	if (len(argv) != 6 and len(argv) != 7):
		usage();
		exit();
	
	source = argv[1];
	type = argv[2];
	time = argv[3];
	node = argv[4];
	target = argv[5];
	is_parse = "";
	if (len(argv) == 7):
		is_parse = argv[6];
	
	if (source == "caida" and type == "ip" and node != "all"):
		dir = "data/caida/ip/"+time+"/"+node+"/";
		file = source+"."+type+"."+time+"."+node+"."+target;
		raw_file = source+"."+type+"."+time+"."+node;
		if not os.path.exists(dir):
			os.makedirs(dir);
		
		if os.path.exists(dir+file):
			print "already exists";
			exit();
		elif not os.path.exists(dir+raw_file):	
			print "raw file does not exist, start downloading...";
			auth = read_auth("caida");
			if ( len(auth) != 2 ):
				print "auth failed";
				exit();

			url = caida.get_url("caida", time, node);
			if ( url == None):
				print "no such record found";
				exit();
	
			download_worker.download_caida_restricted_worker(url, dir, raw_file+".warts.gz", auth[0], auth[1]);
			print "finished downloading.";
			print "dumping...";
			os.system("gunzip -q "+dir+raw_file+".warts.gz");
			os.system("sc_analysis_dump "+dir+raw_file+".warts > "+dir+raw_file); 
			print "finished dumping.";

		if not is_parse == "not":
			print "building topo...";
			topo = ip_topo.topo_graph(ip_topo.get_src(dir+raw_file),True);
			topo.build(dir+raw_file,"caida",True,True);
			topo.disp_stats();
			print "generating map...";
			topo.generate_map();
			print "exporting map...";
			topo.export_map(dir+raw_file);
			print "exporting simplified topo...";
			topo.export_topo_simplified(dir+raw_file);
			print "exporting graphviz...";
			topo.export_graphviz(dir+raw_file);
			print "exporting degree...";
			topo.export_degree(dir+raw_file);
			print "exporting path tree...";
			topo.export_path_tree(dir+raw_file);
			print "exporting border...";
			topo.export_border_ip(dir+raw_file);

	elif (source == "caida" and type == "ip" and node == "all"):
		dir = "data/caida/ip/"+time+"/all/";
		file = source+"."+type+"."+time+".all."+target;
		if not os.path.exists(dir):
			os.makedirs(dir);
		
		if os.path.exists(dir+file):
			print "already exists";
			exit();
		else:
			url_list = caida.get_time_list("caida", time);
			for u in url_list:
				node = u.split('/')[9].split('.')[5];
				raw_file = source+"."+type+"."+time+"."+node;
				if not os.path.exists(dir+raw_file):
					print "raw file "+raw_file+" does not exist, start downloading...";
					auth = read_auth("caida");
					if ( len(auth) != 2 ):
						print "auth failed";
						exit();
		
					url = caida.get_url("caida", time, node);
					if ( url == None):
						print "no such record found";
	
					download_worker.download_caida_restricted_worker(url, dir, raw_file+".warts.gz", auth[0], auth[1]);
					print "finished downloading.";
					print "dumping...";
					os.system("gunzip -q "+dir+raw_file+".warts.gz");
					os.system("sc_analysis_dump "+dir+raw_file+".warts > "+dir+raw_file); 
					print "finished dumping.";

	elif (source == "iplane" and type == "ip"):
		dir = "data/iplane/ip/"+time+"/";
		file = source+"."+type+"."+time+"."+node+"."+target;
		raw_file = source+"."+type+"."+time;
		url = iplane.get_url("iplane", time);
		dir_out = url.split('/')[6].split('.')[0];

		if not os.path.exists(dir):
			os.makedirs(dir);
		
		if os.path.exists(dir+file):
			print "already exists";
			exit();
		if not os.path.exists(dir+raw_file+".tar.gz"):
			print "raw file does not exist, start downloading...";
			auth = read_auth("iplane");
			if ( len(auth) != 2 ):
				print "auth failed";
				exit();

			if ( url == None ):
				print "no such record found";
				exit();
	
			download_worker.download_iplane_restricted_worker(url, dir, raw_file+".tar.gz", auth[0], auth[1]);
			print "finished downloading.";
		
		if not os.path.exists(dir+dir_out):
			print "uncompressing ...";
			os.system("tar -zxvf "+dir+raw_file+".tar.gz"+" -C "+dir);

		if not os.path.exists(dir+raw_file+".ls"):
			os.system("ls "+dir+dir_out+" > "+dir+raw_file+".ls");
			f_ls = open(dir+raw_file+".ls");
	
			for line in f_ls.readlines():
				n = line.split('.',2)[2].strip('\n');
				out_file = raw_file+"."+n
				os.system("./readoutfile "+dir+dir_out+"/"+line.strip('\n')+" > "+dir+out_file);
				print ("./readoutfile "+dir+dir_out+"/"+line.strip('\n')+" > "+dir+out_file);
			
			f_ls.close();
			print "finished dumping.";
		
		is_included = False;
		f_ls = open(dir+raw_file+".ls");

		for line in f_ls.readlines():
			n = line.split('.',2)[2].strip('\n');
			if (n == node):
				is_included = True;
		f_ls.close();

		if(not is_included):
			print "no such record";
			exit();
		
		print "building topo...";
		topo = ip_topo.topo_graph(ip_topo.get_src_iplane(dir+raw_file+"."+node),True);
		topo.build(dir+raw_file+"."+node,"iplane",True,True);
		topo.disp_stats();
		print "generating map...";
		topo.generate_map();
		print "exporting map...";
		topo.export_map(dir+raw_file+"."+node);
		print "exporting simplified topo...";
		topo.export_topo_simplified(dir+raw_file+"."+node);
		print "exporting graphviz...";
		topo.export_graphviz(dir+raw_file+"."+node);
		print "exporting degree...";
		topo.export_degree(dir+raw_file+"."+node);
		print "exporting path tree...";
		topo.export_path_tree(dir+raw_file+"."+node);
		print "exporting border...";
		topo.export_border_ip(dir+raw_file);