################### ################### ## Main ################### force_utf8_hack() ################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" nodes = list_data.load_data(plnode_dir + deploy_node_filename) if DEBUG3: print " %d nodes" % (len(nodes)) ################### ## Get Hostname ################### if DEBUG2: print "Get Existing Hostname" hostnames = set(list_data.load_data(hosts_dir + hostname_filename)) if DEBUG3: print " %d hostnames" % (len(list(hostnames))) IF_DATA_READ = 1
################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" if os.path.exists(plnode_dir + ready_node_filename): filename = plnode_dir + ready_node_filename else: filename = plnode_dir + deploy_node_filename ####### # filename = plnode_dir + deploy_node_filename ####### nodes = list_data.load_data(filename) ################### ## read cname and dns ################### if DEBUG2: print "Read Data" ################################################### ## only CDN: os.system("cat " + cname_dir + cname_filename + " | grep \"cdn\|chinacache\|ccgslb\" > tmp.cdn_list.txt") # os.system("cat " + cname_dir + cname_filename + " | grep \"aka\|chinacache\|ccgslb\" > tmp.cdn_list.txt") cnames = list_data.load_data("tmp.cdn_list.txt") os.system("rm tmp.cdn_list.txt") ## only valid DNS:
################### ## Variables ################### ################### ## Main ################### ################### ## read cname and dns ################### if DEBUG2: print "Read Data" cnames = list_data.load_data(cname_dir + cname_filename) auths = list_data.load_data(cname_dir + auth_filename) if DEBUG3: print " #cnames=%d" % (len(cnames)) if DEBUG3: print " #auth dns=%d" % (len(auths)) ################### ## DNS Query ################### if DEBUG2: print "DNS Query" ips = set() for cname in cnames: for auth in auths:
################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" if os.path.exists(plnode_dir + ready_node_filename): filename = plnode_dir + ready_node_filename else: filename = plnode_dir + deploy_node_filename ####### # filename = plnode_dir + deploy_node_filename ####### nodes = list_data.load_data(filename) ################### ## get data ################### if DEBUG2: print "Get Data" cnames = data.load_data(cname_dir + cname_dict_filename) auths = data.load_data(cname_dir + auth_dict_filename) cname_list = set(list_data.load_data(cname_dir + cname_filename)) auth_list = set(list_data.load_data(cname_dir + auth_filename)) fail_cnames = data.load_data(cname_dir + fail_cname_dict_filename) fail_auths = data.load_data(cname_dir + fail_auth_dict_filename) fail_cname_list = set(list_data.load_data(cname_dir + fail_cname_filename)) fail_auth_list = set(list_data.load_data(cname_dir + fail_auth_filename))
# print located_ips[ip] if 'loc' in located_ips[ip]: print " %s, %s" % (ip, located_ips[ip]['loc']) else: print " %s\n " % (ip) print located_ips[ip] # exit() ################### ## read ips ################### if DEBUG2: print "Read IPs" if os.path.exists(PARAM_FILE): remain_ips = set(list_data.load_data(PARAM_FILE)) else: ips = data.load_data(ips_dir + ip_dict_filename) remain_ips = set([]) for cname in ips: for dns in ips[cname]: for ip in ips[cname][dns]: if ip not in located_ips: remain_ips.add(ip) remain_ips = list(remain_ips) if DEBUG3: print " #remainning ips: %d" % (len(remain_ips)) ################### ## Locate IP
################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" if os.path.exists(plnode_dir + ready_node_filename): filename = plnode_dir + ready_node_filename else: filename = plnode_dir + deploy_node_filename ####### filename = plnode_dir + deploy_node_filename ####### nodes = list_data.load_data(filename) if DEBUG3: print " %d nodes" % (len(nodes)) ################### ## Get Entrance ################### if DEBUG2: print "Get Entrance" entrances = list_data.load_data(hosts_dir + entrance_filename) if DEBUG3: print " %d entrances" % (len(entrances)) IF_DATA_READ = 1 ################### ## Generate Parameter Files ###################
################### force_utf8_hack() os.system("rm %s" % (DONE_IND_FILE)) os.system("rm %s" % (KILLED_IND_FILE)) os.system("touch %s" % (RUNNING_IND_FILE)) ################### ## read dns candidate ################### if DEBUG2: print "Read DNS Candidate" cname = "www.baidu.com" if os.path.exists(PARAM_FILE): auths = list_data.load_data(PARAM_FILE) else: auths = list_data.load_data(cname_dir + auth_filename) if DEBUG3: print " #auth dns=%d" % (len(auths)) ################### ## read done DNS ################### if DEBUG2: print "Read Done DNS" valid_dns = set(list_data.load_data(dns_dir + dns_filename)) invalid_dns = set(list_data.load_data(dns_dir + bad_dns_filename)) IF_DATA_READ = 1
################### ## DEBUG # list_data.store_data("./tmp.txt", map(str, [11, 22])) # exit() ################### ################### ## Main ################### ################### ## Load Data ################### if DEBUG2: print "Load Data" hostnames = list_data.load_data(hostname_file) cnames = list_data.load_data(cname_file) auths = list_data.load_data(auth_file) valid_dns = list_data.load_data(valid_dns_file) ips = data.load_data(ip_data_file) geos = data.load_data(ip_geo_data_file) located_ips = data.load_data(exist_ipgeo_file) if DEBUG3: print " #hosts=%d" % (len(hostnames)) if DEBUG3: print " #cnames=%d" % (len(cnames)) if DEBUG3: print " #auth dns=%d" % (len(auths)) if DEBUG3: print " #valid dns=%d" % (len(valid_dns)) if DEBUG3: print " #cdn with ip=%d" % (len(ips)) if DEBUG3: print " #cdn providers=%d" % (len(geos)) if DEBUG3: print " #located IPs=%d" % (len(located_ips))
################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" if os.path.exists(plnode_dir + ready_node_filename): filename = plnode_dir + ready_node_filename else: filename = plnode_dir + deploy_node_filename ####### filename = plnode_dir + deploy_node_filename ####### nodes = list_data.load_data(filename) if DEBUG3: print " %d nodes" % (len(nodes)) ################### ## Get Entrance ################### if DEBUG2: print "Get Entrance" entrances = list_data.load_data(hosts_dir + entrance_filename) if DEBUG3: print " %d entrances" % (len(entrances)) IF_DATA_READ = 1 ###################
################### ################### ## Main ################### force_utf8_hack() ################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" nodes = list_data.load_data(plnode_dir + deploy_node_filename) ################### ## read done DNS ################### if DEBUG2: print "Read Done DNS" valid_dns = set(list_data.load_data(dns_dir + dns_filename)) invalid_dns = set(list_data.load_data(dns_dir + bad_dns_filename)) IF_DATA_READ = 1 if DEBUG3: print " # done valid dns=%d" % (len(valid_dns)) if DEBUG3:
(lang, enc) = locale.getlocale(aref) if lang != None: try: locale.setlocale(aref, (lang, 'UTF-8')) except: os.environ[attr] = lang + '.UTF-8' ################### ## Main ################### force_utf8_hack() cnames = data.load_data(output_dir + cname_dict_filename) auths = data.load_data(output_dir + auth_dict_filename) cname_list = set(list_data.load_data(output_dir + cname_filename)) auth_list = set(list_data.load_data(output_dir + auth_filename)) fail_cnames = data.load_data(output_dir + fail_cname_dict_filename) fail_auths = data.load_data(output_dir + fail_auth_dict_filename) fail_cname_list = set(list_data.load_data(output_dir + fail_cname_filename)) fail_auth_list = set(list_data.load_data(output_dir + fail_auth_filename)) # data_cname_list = set() # for hostname in cnames: # # print " " + hostname # data_cname_list.update(cnames[hostname]) # print " # hostnames = %d" % (len(cnames)) # print " # cnames = %d" % (len(data_cname_list)) # print " # cnames list = %d" % (len(cname_list))
log_name = 'fs_usage' if file_type == "app": program_names = [ "PowerPoint", "Word", "Excel", "Chrome", #"Firefox", "Safari", "Skype", "iTunes", "VLC", "QuickTime" ] elif file_type == "web": program_names = list_data.load_data("../collect/websites/websites.txt") else: sys.exit('Example: ./preprocess_fs_usage.py <FILE NAME> <FILE TYPE>') for program_name in program_names: if len(program_name) < 1: continue print ("Current program name: %s\n" %program_name) f1 = open(input_dir + file_name + "." + program_name + "." + log_name + ".processed.time.txt", 'w') f2 = open(input_dir + file_name + "." + program_name + "." + log_name + ".processed.cmd.txt", 'w') with open(input_dir + file_name + "." + program_name + "." + log_name + ".processed.csv", 'rb') as csvfile: spamreader = csv.reader(csvfile) spamreader = list(spamreader) cmd_type = dict() app_type = dict() cmd_type_cnt = 0
################### ## Main ################### force_utf8_hack() os.system("rm %s" % (DONE_IND_FILE)) os.system("rm %s" % (KILLED_IND_FILE)) os.system("touch %s" % (RUNNING_IND_FILE)) ################### ## read cname and dns ################### if DEBUG2: print "Read Data" if os.path.exists(PARAM_FILE_CNAME): cnames = list_data.load_data(PARAM_FILE_CNAME) else: cnames = list_data.load_data(cname_dir + cname_filename) if os.path.exists(PARAM_FILE_AUTH): auths = list_data.load_data(PARAM_FILE_AUTH) elif os.path.exists(dns_dir + dns_filename): auths = list_data.load_data(dns_dir + dns_filename) else: auths = list_data.load_data(cname_dir + auth_filename) if DEBUG3: print " #cnames=%d" % (len(cnames)) if DEBUG3: print " #auth dns=%d" % (len(auths)) ################### ## read done IPs
# print located_ips[ip] if 'loc' in located_ips[ip]: print " %s, %s" % (ip, located_ips[ip]['loc']) else: print " %s\n " % (ip) print located_ips[ip] # exit() ################### ## read ips ################### if DEBUG2: print "Read IPs" if os.path.exists(PARAM_FILE): remain_ips = set(list_data.load_data(PARAM_FILE)) else: ips = data.load_data(ips_dir + ip_dict_filename) remain_ips = set([]) for cname in ips: for dns in ips[cname]: for ip in ips[cname][dns]: if ip not in located_ips: remain_ips.add(ip) remain_ips = list(remain_ips) if DEBUG3: print " #remainning ips: %d" % (len(remain_ips)) ################### ## Locate IP ###################
################### ## Main ################### force_utf8_hack() ################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" if os.path.exists(plnode_dir + ready_node_filename): filename = plnode_dir + ready_node_filename else: filename = plnode_dir + deploy_node_filename nodes = list_data.load_data(filename) ################### ## Read IP to Geolocation Database ################### if DEBUG2: print "Read IP to Geolocation Database" located_ips = data.load_data(geo_db_dir + ipinfo_db_filename) IF_DATA_READ = 1 if DEBUG3: print " #records=%d" % (len(located_ips)) ################### ## read ips
## DEBUG # list_data.store_data("./tmp.txt", map(str, [11, 22])) # exit() ################### ################### ## Main ################### ################### ## Load Data ################### if DEBUG2: print "Load Data" hostnames = list_data.load_data(hostname_file) cnames = list_data.load_data(cname_file) auths = list_data.load_data(auth_file) valid_dns = list_data.load_data(valid_dns_file) ips = data.load_data(ip_data_file) geos = data.load_data(ip_geo_data_file) if DEBUG3: print " #hosts=%d" % (len(hostnames)) if DEBUG3: print " #cnames=%d" % (len(cnames)) if DEBUG3: print " #auth dns=%d" % (len(auths)) if DEBUG3: print " #valid dns=%d" % (len(valid_dns)) if DEBUG3: print " #cdn with ip=%d" % (len(ips)) if DEBUG3: print " #located ips=%d" % (len(geos))
## DEBUG # exit() ################### ################### ## Main ################### force_utf8_hack() ################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" nodes = list_data.load_data(plnode_dir + deploy_node_filename) ################### ## get data ################### if DEBUG2: print "Get Existing IP Data" ips = data.load_data(ips_dir + ip_dict_filename) IF_DATA_READ = 1 if DEBUG3: print " #cnames=%d" % (len(ips)) # a = input(' ') ###################
################### ## DEBUG # exit() ################### ################### ## Main ################### force_utf8_hack() ################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" nodes = list_data.load_data(plnode_dir + deploy_node_filename) if DEBUG3: print " %d nodes" % (len(nodes)) ################### ## get data ################### if DEBUG2: print "Get Data" cnames = data.load_data(cname_dir + cname_dict_filename) auths = data.load_data(cname_dir + auth_dict_filename) cname_list = set(list_data.load_data(cname_dir + cname_filename)) auth_list = set(list_data.load_data(cname_dir + auth_filename)) fail_cnames = data.load_data(cname_dir + fail_cname_dict_filename) fail_auths = data.load_data(cname_dir + fail_auth_dict_filename) fail_cname_list = set(list_data.load_data(cname_dir + fail_cname_filename)) fail_auth_list = set(list_data.load_data(cname_dir + fail_auth_filename))
################### ## Main ################### force_utf8_hack() os.system("rm %s" % (DONE_IND_FILE)) os.system("rm %s" % (KILLED_IND_FILE)) os.system("touch %s" % (RUNNING_IND_FILE)) ################### ## Read Existing Hostnames ################### if DEBUG2: print "Read Existing Hostnames" hostnames = set(list_data.load_data(hosts_dir + hostname_filename)) IF_DATA_READ = 1 if DEBUG3: print " #hostnames: %d" % (len(hostnames)) ################### ## read entrance websites ################### if DEBUG2: print "Read Entrance Web Sites" if os.path.exists(PARAM_FILE): entrances = list_data.load_data(PARAM_FILE) else: entrances = list_data.load_data(hosts_dir + entrance_filename) if DEBUG3: print " #entrances: %d" % (len(entrances))
################### ## Main ################### force_utf8_hack() os.system("rm %s" % (DONE_IND_FILE)) os.system("rm %s" % (KILLED_IND_FILE)) os.system("touch %s" % (RUNNING_IND_FILE)) ################### ## read cname and dns ################### if DEBUG2: print "Read Data" if os.path.exists(PARAM_FILE_CNAME): cnames = list_data.load_data(PARAM_FILE_CNAME) else: cnames = list_data.load_data(cname_dir + cname_filename) if os.path.exists(PARAM_FILE_AUTH): auths = list_data.load_data(PARAM_FILE_AUTH) elif os.path.exists(dns_dir + dns_filename): auths = list_data.load_data(dns_dir + dns_filename) else: auths = list_data.load_data(cname_dir + auth_filename) if DEBUG3: print " #cnames=%d" % (len(cnames)) if DEBUG3: print " #auth dns=%d" % (len(auths)) ###################
def store_output_files(): # print "Store!" hostnames.update(set(list_data.load_data(hosts_dir + hostname_filename))) list_data.store_data(hosts_dir + hostname_filename, list(hostnames))
################### ## Main ################### force_utf8_hack() ################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" if os.path.exists(plnode_dir + ready_node_filename): filename = plnode_dir + ready_node_filename else: filename = plnode_dir + deploy_node_filename nodes = list_data.load_data(filename) ################### ## Read IP to Geolocation Database ################### if DEBUG2: print "Read IP to Geolocation Database" located_ips = data.load_data(geo_db_dir + ipinfo_db_filename) IF_DATA_READ = 1 if DEBUG3: print " #records=%d" % (len(located_ips)) ################### ## read ips ################### if DEBUG2: print "Read IPs"
################### ## DEBUG # exit() ################### ################### ## Main ################### force_utf8_hack() ################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" nodes = list_data.load_data(plnode_dir + deploy_node_filename) ################### ## Read IP to Geolocation Database ################### if DEBUG2: print "Read IP to Geolocation Database" located_ips = data.load_data(geo_db_dir + ipinfo_db_filename) IF_DATA_READ = 1 if DEBUG3: print " #records=%d" % (len(located_ips)) ################### ## Check Remote IPs -- Prepare Tmp Directory ################### if DEBUG2: print "Prepare Tmp Directory"
os.environ[attr] = lang + '.UTF-8' if __name__ == "__main__": if len(sys.argv) < 4: sys.exit('UsageL ./visit_websites.py <FileName> <IterationLoop> <startNum> <WebNum>') sys.exit('Example: ./visit_websites.py 20160616.exp01 1 1 20') fileName = sys.argv[1] iterNum = int(sys.argv[2]) startNum = int(sys.argv[3]) webNum = int(sys.argv[4]) force_utf8_hack() if DEBUG2: print "Load Web Sites" websites = list_data.load_data(input_dir + filename) if DEBUG2: print " # web sites: %d" % (len(websites)) if DEBUG2: print "Visit web sites in turns" open_file = "%s/%s.web_time.txt" %(outputDir,fileName) #if DEBUG2: print "Begin system call tracking" #logFile = "%r.pcap" %fileName #cmd = "sudo ./flow_track.sh %r %r" %(outputDir, logFile) #os.system(cmd) with open(open_file, "w") as myfile: for run in xrange(0, iterNum): # for wi in xrange(0,len(websites)): for wi in xrange(0+startNum-1, webNum+startNum-1):
signal.signal(signal.SIGALRM, timeout_handler) ################### ## Variables ################### ################### ## Main ################### ################### ## read cname and dns ################### if DEBUG2: print "Read Data" cnames = list_data.load_data(cname_dir + cname_filename) auths = list_data.load_data(cname_dir + auth_filename) if DEBUG3: print " #cnames=%d" % (len(cnames)) if DEBUG3: print " #auth dns=%d" % (len(auths)) ################### ## DNS Query ################### if DEBUG2: print "DNS Query" ips = set() for cname in cnames: for auth in auths: # cname = "ghs.google.com"
################### ## DEBUG # exit() ################### ################### ## Main ################### force_utf8_hack() ################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" nodes = list_data.load_data(plnode_dir + deploy_node_filename) ################### ## read done DNS ################### if DEBUG2: print "Read Done DNS" valid_dns = set(list_data.load_data(dns_dir + dns_filename)) invalid_dns = set(list_data.load_data(dns_dir + bad_dns_filename)) IF_DATA_READ = 1 if DEBUG3: print " # done valid dns=%d" % (len(valid_dns)) if DEBUG3: print " # done invalid dns=%d" % (len(invalid_dns)) ################### ## Check Remote Status -- Prepare Tmp Directory
# exit() ################### ################### ## Main ################### force_utf8_hack() ################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" nodes = list_data.load_data(plnode_dir + deploy_node_filename) if DEBUG3: print " %d nodes" % (len(nodes)) ################### ## get data ################### if DEBUG2: print "Get Data" cnames = data.load_data(cname_dir + cname_dict_filename) auths = data.load_data(cname_dir + auth_dict_filename) cname_list = set(list_data.load_data(cname_dir + cname_filename)) auth_list = set(list_data.load_data(cname_dir + auth_filename)) fail_cnames = data.load_data(cname_dir + fail_cname_dict_filename) fail_auths = data.load_data(cname_dir + fail_auth_dict_filename) fail_cname_list = set(list_data.load_data(cname_dir + fail_cname_filename))
################### ## get PlanetLab nodes states ################### if DEBUG2: print "Get PlanetLab Nodes" if os.path.exists(plnode_dir + ready_node_filename): filename = plnode_dir + ready_node_filename else: filename = plnode_dir + deploy_node_filename ####### # filename = plnode_dir + deploy_node_filename ####### nodes = list_data.load_data(filename) ################### ## read dns candidate ################### if DEBUG2: print "Read DNS Candidate" auths = set(list_data.load_data(cname_dir + auth_filename)) if DEBUG3: print " #auth dns=%d" % (len(auths)) ################### ## read done DNS ###################