def analyze_date_caida(self):
date_dir = self.data_dir + "/" + self.date
if (not os.path.exists(date_dir) or not os.listdir(date_dir)):
print("path is empty or not existent")
exit()
for fn in os.listdir(date_dir):
if (re.findall(".*\.gz$", fn)):
print "analyzing " + fn + " ..."
if (os.path.exists(date_dir + "/" + fn.rstrip(".gz"))):
os.system("rm -f " + date_dir + "/" + fn.rstrip(".gz"))
warts_name = date_dir + "/" + fn.strip(".gz")
dump_name = date_dir + "/" + warts_name.split('.')[-2]
monitor_name = warts_name.split('.')[-2]
temp_topo = ip_topo.topo_graph()
temp_topo.build_popen_pipeline(date_dir, fn)
self.topo.merge(temp_topo, monitor_name)
print fn + " analyzed"
print self.topo.get_node_num()
print self.topo.get_target_num()
print "exporting topo to db"
self.export_topo_to_db()
print "finished exporting topo to db"
def __init__(self, date, file_name=""):
self.config = config.get_config_section_dict("config.ini", "data")
self.data_dir = self.config["data_dir"]
self.date = date
self.topo = ip_topo.topo_graph()
self.db_helper = db_helper.db_helper()
def analyze_date_caida(self):
date_dir = self.data_dir + "/" + self.date
if (not os.path.exists(date_dir) or not os.listdir(date_dir)):
print("path is empty or not existent")
exit()
for fn in os.listdir(date_dir):
if (re.findall(".*\.gz$", fn)):
print "analyzing " + fn + " ..."
if (os.path.exists(date_dir + "/" + fn.rstrip(".gz"))):
os.system("rm -f " + date_dir + "/" + fn.rstrip(".gz"))
os.system("gzip -d -k -q " + date_dir + "/" + fn)
warts_name = date_dir + "/" + fn.strip(".gz")
dump_name = date_dir + "/" + warts_name.split('.')[-2]
monitor_name = warts_name.split('.')[-2]
os.system("sc_analysis_dump " + warts_name + " > " + dump_name)
temp_topo = ip_topo.topo_graph()
temp_topo.build(dump_name)
self.topo.merge(temp_topo, monitor_name)
os.system("rm -f " + warts_name)
os.system("rm -f " + dump_name)
print fn + " analyzed"
print self.topo.get_node_num()
print self.topo.get_target_num()
#self.topo.build_networkx_graph()
#print "pickling topo"
#self.pickle_topo()
#print "finished pickling topo"
print "exporting topo to db"
self.export_topo_to_db()
print "finished exporting topo to db"
def analyze_date_caida_pipeline(self):
self.topo = ip_topo.topo_graph()
self.topo.build_pipeline()
print self.topo.get_node_num(), self.topo.get_target_num()
print "exporting topo to db"
self.export_topo_to_db()
print "finished exporting topo to db"
file_08 = []
#file_12 = [];
file_16 = []
print("building topo_08")
url_list = caida.get_time_list("caida", "20080617")
dir = "data/caida/ip/20080617/all/"
for u in url_list:
time = u.split('/')[9].split('.')[4]
node = u.split('/')[9].split('.')[5]
if (time == "20080617"):
raw_file = "caida.ip." + time + "." + node
file_08.append(raw_file)
topo_08 = ip_topo.topo_graph(ip_topo.get_src(dir + file_08[0]), False)
topo_08.build(dir + file_08[0], "caida", False, False)
for i in range(1, len(file_08)):
topo = ip_topo.topo_graph(ip_topo.get_src(dir + file_08[i]), False)
topo.build(dir + file_08[i], "caida", False, False)
topo_08.merge(topo)
print str(i + 1) + " of " + str(len(file_08)) + " done."
'''
print("building topo_12");
url_list = caida.get_time_list("caida","20120617");
dir = "data/caida/ip/20120617/all/";
for u in url_list:
time = u.split('/')[9].split('.')[4];
node = u.split('/')[9].split('.')[5];
if(time=="20120617"):
raw_file = "caida.ip."+time+"."+node;
import ip_topo
topo = ip_topo.topo_graph(ip_topo.get_src_lg(),False);
topo.build("traces.txt","lg",False,False);
topo.calc_deg();
topo.calc_knn();
topo.draw_deg("lg");
topo.draw_knn("lg");
topo.draw_path("lg");
file_08 = [];
#file_12 = [];
file_16 = [];
print("building topo_08");
url_list = caida.get_time_list("caida","20080617");
dir = "data/caida/ip/20080617/all/";
for u in url_list:
time = u.split('/')[9].split('.')[4];
node = u.split('/')[9].split('.')[5];
if(time=="20080617"):
raw_file = "caida.ip."+time+"."+node;
file_08.append(raw_file);
topo_08 = ip_topo.topo_graph(ip_topo.get_src(dir+file_08[0]),False);
topo_08.build(dir+file_08[0],"caida",False,False);
for i in range(1, len(file_08)):
topo = ip_topo.topo_graph(ip_topo.get_src(dir+file_08[i]),False);
topo.build(dir+file_08[i],"caida",False,False);
topo_08.merge(topo);
print str(i+1)+" of "+str(len(file_08))+" done.";
'''
print("building topo_12");
url_list = caida.get_time_list("caida","20120617");
dir = "data/caida/ip/20120617/all/";
for u in url_list:
time = u.split('/')[9].split('.')[4];
node = u.split('/')[9].split('.')[5];
if(time=="20120617"):
def main(argv):
if (len(argv) != 6 and len(argv) != 7):
usage();
exit();
source = argv[1];
type = argv[2];
time = argv[3];
node = argv[4];
target = argv[5];
is_parse = "";
if (len(argv) == 7):
is_parse = argv[6];
if (source == "caida" and type == "ip" and node != "all"):
dir = "data/caida/ip/"+time+"/"+node+"/";
file = source+"."+type+"."+time+"."+node+"."+target;
raw_file = source+"."+type+"."+time+"."+node;
if not os.path.exists(dir):
os.makedirs(dir);
if os.path.exists(dir+file):
print "already exists";
exit();
elif not os.path.exists(dir+raw_file):
print "raw file does not exist, start downloading...";
auth = read_auth("caida");
if ( len(auth) != 2 ):
print "auth failed";
exit();
url = caida.get_url("caida", time, node);
if ( url == None):
print "no such record found";
exit();
download_worker.download_caida_restricted_worker(url, dir, raw_file+".warts.gz", auth[0], auth[1]);
print "finished downloading.";
print "dumping...";
os.system("gunzip -q "+dir+raw_file+".warts.gz");
os.system("sc_analysis_dump "+dir+raw_file+".warts > "+dir+raw_file);
print "finished dumping.";
if not is_parse == "not":
print "building topo...";
topo = ip_topo.topo_graph(ip_topo.get_src(dir+raw_file),True);
topo.build(dir+raw_file,"caida",True,True);
topo.disp_stats();
print "generating map...";
topo.generate_map();
print "exporting map...";
topo.export_map(dir+raw_file);
print "exporting simplified topo...";
topo.export_topo_simplified(dir+raw_file);
print "exporting graphviz...";
topo.export_graphviz(dir+raw_file);
print "exporting degree...";
topo.export_degree(dir+raw_file);
print "exporting path tree...";
topo.export_path_tree(dir+raw_file);
print "exporting border...";
topo.export_border_ip(dir+raw_file);
elif (source == "caida" and type == "ip" and node == "all"):
dir = "data/caida/ip/"+time+"/all/";
file = source+"."+type+"."+time+".all."+target;
if not os.path.exists(dir):
os.makedirs(dir);
if os.path.exists(dir+file):
print "already exists";
exit();
else:
url_list = caida.get_time_list("caida", time);
for u in url_list:
node = u.split('/')[9].split('.')[5];
raw_file = source+"."+type+"."+time+"."+node;
if not os.path.exists(dir+raw_file):
print "raw file "+raw_file+" does not exist, start downloading...";
auth = read_auth("caida");
if ( len(auth) != 2 ):
print "auth failed";
exit();
url = caida.get_url("caida", time, node);
if ( url == None):
print "no such record found";
download_worker.download_caida_restricted_worker(url, dir, raw_file+".warts.gz", auth[0], auth[1]);
print "finished downloading.";
print "dumping...";
os.system("gunzip -q "+dir+raw_file+".warts.gz");
os.system("sc_analysis_dump "+dir+raw_file+".warts > "+dir+raw_file);
print "finished dumping.";
elif (source == "iplane" and type == "ip"):
dir = "data/iplane/ip/"+time+"/";
file = source+"."+type+"."+time+"."+node+"."+target;
raw_file = source+"."+type+"."+time;
url = iplane.get_url("iplane", time);
dir_out = url.split('/')[6].split('.')[0];
if not os.path.exists(dir):
os.makedirs(dir);
if os.path.exists(dir+file):
print "already exists";
exit();
if not os.path.exists(dir+raw_file+".tar.gz"):
print "raw file does not exist, start downloading...";
auth = read_auth("iplane");
if ( len(auth) != 2 ):
print "auth failed";
exit();
if ( url == None ):
print "no such record found";
exit();
download_worker.download_iplane_restricted_worker(url, dir, raw_file+".tar.gz", auth[0], auth[1]);
print "finished downloading.";
if not os.path.exists(dir+dir_out):
print "uncompressing ...";
os.system("tar -zxvf "+dir+raw_file+".tar.gz"+" -C "+dir);
if not os.path.exists(dir+raw_file+".ls"):
os.system("ls "+dir+dir_out+" > "+dir+raw_file+".ls");
f_ls = open(dir+raw_file+".ls");
for line in f_ls.readlines():
n = line.split('.',2)[2].strip('\n');
out_file = raw_file+"."+n
os.system("./readoutfile "+dir+dir_out+"/"+line.strip('\n')+" > "+dir+out_file);
print ("./readoutfile "+dir+dir_out+"/"+line.strip('\n')+" > "+dir+out_file);
f_ls.close();
print "finished dumping.";
is_included = False;
f_ls = open(dir+raw_file+".ls");
for line in f_ls.readlines():
n = line.split('.',2)[2].strip('\n');
if (n == node):
is_included = True;
f_ls.close();
if(not is_included):
print "no such record";
exit();
print "building topo...";
topo = ip_topo.topo_graph(ip_topo.get_src_iplane(dir+raw_file+"."+node),True);
topo.build(dir+raw_file+"."+node,"iplane",True,True);
topo.disp_stats();
print "generating map...";
topo.generate_map();
print "exporting map...";
topo.export_map(dir+raw_file+"."+node);
print "exporting simplified topo...";
topo.export_topo_simplified(dir+raw_file+"."+node);
print "exporting graphviz...";
topo.export_graphviz(dir+raw_file+"."+node);
print "exporting degree...";
topo.export_degree(dir+raw_file+"."+node);
print "exporting path tree...";
topo.export_path_tree(dir+raw_file+"."+node);
print "exporting border...";
topo.export_border_ip(dir+raw_file);