def difference(old_folder_name,new_folder_name): list=get_list() remove_ip={} add_ip={} for file_name in list: file_to_read=open(str(new_folder_name)+"/"+file_name,'r') for line in file_to_read: if "#" in line: if "Date:" in line: data=line.split('Date:') time_in_epoch=calendar.timegm(datetime.fromtimestamp(mktime(time.strptime(data[1].strip(),"%a %b %d %H:%M:%S %Z %Y"))).timetuple()) continue with open(str(old_folder_name)+"/"+file_name) as f: temp=[] for line in f: if '#' in line: continue temp.append(line.strip()) t1s = set(temp) with open(str(new_folder_name)+'/'+file_name) as f: temp=[] for line in f: if '#' in line: continue temp.append(line.strip()) t2s = set(temp) for diff in t1s-t2s: remove_ip[diff]=time_in_epoch for diff in t2s-t1s: add_ip[diff]=time_in_epoch return add_ip,remove_ip
def download_files(folder_name,download): #https://raw.githubusercontent.com/ktsaou/blocklist-ipsets/master/ri_connect_proxies_30d.ipset list=get_list() os.system('mkdir '+str(folder_name)) if download: for item in list: print "Downloading ",item,str(list.index(item))," of ",str(len(list)) url="https://raw.githubusercontent.com/ktsaou/blocklist-ipsets/master/"+item file_downloaded = urllib2.urlopen(url) output = open(str(folder_name)+'/'+item,'w') output.write(file_downloaded.read()) output.close()
def get_ip(folder_name): list=get_list() ip_list={} for file_name in list: if "set_file" in file_name: continue file_to_read=open(str(folder_name)+'/'+str(file_name),'r') for line in file_to_read: if "#" in line: if "Date:" in line: data=line.split('Date:') time_in_epoch=calendar.timegm(datetime.fromtimestamp(mktime(time.strptime(data[1].strip(),"%a %b %d %H:%M:%S %Z %Y"))).timetuple()) continue else: ip_address=line.strip() ip_list[ip_address]=time_in_epoch return ip_list
def init_ip(folder_name): total_ips=0 date_counter=0 compare_list=[] file_count=0 list=get_list() cur.execute('USE SENSS') insert_count=0; cmd='' #IP Addresses. for file_name in list: file_count=file_count+1 if "set_file" in file_name: continue file_to_read=open(str(folder_name)+'/'+str(file_name),'r') count=0 print "Doing file-",file_count,file_name for line in file_to_read: count=count+1 if "#" in line: if "Date:" in line: data=line.split('Date:') time_in_epoch=calendar.timegm(datetime.fromtimestamp(mktime(time.strptime(data[1].strip(),"%a %b %d %H:%M:%S %Z %Y"))).timetuple()) date_counter=date_counter+1 compare_list.append(file_name) continue else: insert_count=insert_count+1 ip_address=line.strip() if insert_count%10000==1: cmd='INSERT INTO IP(ID,FILE_NAME,IP_ADDRESS,START_TIME,NUMBER_OF_APPEARANCE,ACTIVE)VALUES(NOT NULL,\''+str(file_name)+'\',\''+str(ip_address)+'\',\''+str(time_in_epoch)+'\',1,1)' else: cmd=cmd+","+'(NOT NULL,\''+str(file_name)+'\',\''+str(ip_address)+'\',\''+str(time_in_epoch)+'\',1,1)' if insert_count%10000==0: cur.execute(cmd) db.commit() print "Done with "+str(total_ips)+" /3437022" total_ips=total_ips+1 if insert_count%10000!=0: cur.execute(cmd) db.commit() #SUBNET print total_ips
def compare_and_update(old_folder,new_folder): list=get_list() count=0 cur.execute('USE SENSS') new_ip=0 old_ip=0 total_add=0 total_remove=0 for file_name in list: file_to_read=open(str(new_folder)+"/"+file_name,'r') print "Doing",file_name for line in file_to_read: if "#" in line: if "Date:" in line: data=line.split('Date:') time_in_epoch=calendar.timegm(datetime.fromtimestamp(mktime(time.strptime(data[1].strip(),"%a %b %d %H:%M:%S %Z %Y"))).timetuple()) continue with open(str(old_folder)+"/"+file_name) as f: temp=[] for line in f: if '#' in line: continue temp.append(line.strip()) t1s = set(temp) with open(str(new_folder)+'/'+file_name) as f: temp=[] for line in f: if '#' in line: continue temp.append(line.strip()) t2s = set(temp) remove_ip=[] add_ip=[] for diff in t1s-t2s: remove_ip.append(diff) for diff in t2s-t1s: add_ip.append(diff) #ID #FILE_NAME #IP_ADDRESS #NUMBER_OF_APPEARANCE #START_TIME #DURATION #ACTIVE insert_count=0 for ip in add_ip: cur.execute('SELECT ID FROM IP WHERE IP_ADDRESS=%s and FILE_NAME=%s',(str(ip),str(file_name))) if not cur.rowcount: insert_count=insert_count+1 ip_address=line.strip() if insert_count%10000==1: cmd='INSERT INTO IP(ID,FILE_NAME,IP_ADDRESS,START_TIME,NUMBER_OF_APPEARANCE,ACTIVE)VALUES(NOT NULL,\''+str(file_name)+'\',\''+str(ip_address)+'\',\''+str(time_in_epoch)+'\',1,1)' else: cmd=cmd+","+'(NOT NULL,\''+str(file_name)+'\',\''+str(ip_address)+'\',\''+str(time_in_epoch)+'\',1,1)' if insert_count%10000==0: cur.execute(cmd) db.commit() else: for item in cur: cur.execute('UPDATE IP SET START_TIME=%s,ACTIVE=1 WHERE ID=%s',(str(time_in_epoch),str(item[0]))) db.commit() if insert_count%10000!=0: cur.execute(cmd) db.commit() for ip in remove_ip: cur.execute('SELECT ID,START_TIME,NUMBER_OF_APPEARANCE,DURATION FROM IP WHERE IP_ADDRESS=%s AND FILE_NAME=%s',(str(ip),str(file_name))) for item in cur.fetchall(): id=int(item[0]) start_time=int(item[1]) number_of_appearance=int(item[2]) number_of_appearance=number_of_appearance+1 end_time=int(calendar.timegm(time.gmtime())) duration_string=str(item[3]) duration=int(end_time)-start_time duration_string=duration_string+","+str(duration) cur.execute("UPDATE IP SET NUMBER_OF_APPEARANCE=%s,DURATION=%s,ACTIVE=0 WHERE ID=%s",(str(number_of_appearance),str(duration_string),str(id))) db.commit() print total_add,total_remove return old_ip,new_ip