Esempio n. 1
0
def difference(old_folder_name,new_folder_name):
	list=get_list()
	remove_ip={}
	add_ip={}
	for file_name in list:
		file_to_read=open(str(new_folder_name)+"/"+file_name,'r')
		for line in file_to_read:
			if "#" in line:
				if "Date:" in line:
					data=line.split('Date:')
					time_in_epoch=calendar.timegm(datetime.fromtimestamp(mktime(time.strptime(data[1].strip(),"%a %b  %d %H:%M:%S %Z %Y"))).timetuple())
				continue
		
		with open(str(old_folder_name)+"/"+file_name) as f:
			temp=[]
			for line in f:
				if '#' in line:
					continue
				temp.append(line.strip())
			t1s = set(temp)
		
		with open(str(new_folder_name)+'/'+file_name) as f:
			temp=[]
			for line in f:
				if '#' in line:
					continue
				temp.append(line.strip())
			t2s = set(temp)
		
		for diff in t1s-t2s:
			remove_ip[diff]=time_in_epoch

		for diff in t2s-t1s:
			add_ip[diff]=time_in_epoch
	return add_ip,remove_ip
def download_files(folder_name,download):
	#https://raw.githubusercontent.com/ktsaou/blocklist-ipsets/master/ri_connect_proxies_30d.ipset
	list=get_list()
	os.system('mkdir '+str(folder_name))
	if download:
		for item in list:
			print "Downloading ",item,str(list.index(item))," of ",str(len(list))
			url="https://raw.githubusercontent.com/ktsaou/blocklist-ipsets/master/"+item
			file_downloaded = urllib2.urlopen(url)
			output = open(str(folder_name)+'/'+item,'w')
			output.write(file_downloaded.read())
			output.close()
Esempio n. 3
0
def get_ip(folder_name):
	list=get_list()
	ip_list={}
	for file_name in list:
		if "set_file" in file_name:
			continue
		file_to_read=open(str(folder_name)+'/'+str(file_name),'r')
		for line in file_to_read:
			if "#" in line:
				if "Date:" in line:
					data=line.split('Date:')
					time_in_epoch=calendar.timegm(datetime.fromtimestamp(mktime(time.strptime(data[1].strip(),"%a %b  %d %H:%M:%S %Z %Y"))).timetuple())
					continue
			else:
				ip_address=line.strip()
				ip_list[ip_address]=time_in_epoch
	return ip_list
Esempio n. 4
0
def init_ip(folder_name):
	total_ips=0
	date_counter=0
	compare_list=[]
	file_count=0
	list=get_list()
	cur.execute('USE SENSS')
	insert_count=0;
	cmd=''
	#IP Addresses.
	for file_name in list:
		file_count=file_count+1
		if "set_file" in file_name:
			continue
		file_to_read=open(str(folder_name)+'/'+str(file_name),'r')
		count=0
		print "Doing file-",file_count,file_name
		for line in file_to_read:
			count=count+1
			if "#" in line:
				if "Date:" in line:
					data=line.split('Date:')
					time_in_epoch=calendar.timegm(datetime.fromtimestamp(mktime(time.strptime(data[1].strip(),"%a %b  %d %H:%M:%S %Z %Y"))).timetuple())
					date_counter=date_counter+1	
					compare_list.append(file_name)
				continue
			else:
				insert_count=insert_count+1
				ip_address=line.strip()
				if insert_count%10000==1:
					cmd='INSERT INTO IP(ID,FILE_NAME,IP_ADDRESS,START_TIME,NUMBER_OF_APPEARANCE,ACTIVE)VALUES(NOT NULL,\''+str(file_name)+'\',\''+str(ip_address)+'\',\''+str(time_in_epoch)+'\',1,1)'
				else:
					cmd=cmd+","+'(NOT NULL,\''+str(file_name)+'\',\''+str(ip_address)+'\',\''+str(time_in_epoch)+'\',1,1)'
				if insert_count%10000==0:
					cur.execute(cmd)
					db.commit()
					print "Done with "+str(total_ips)+" /3437022"
			total_ips=total_ips+1
	if insert_count%10000!=0:
		cur.execute(cmd)
		db.commit()
	

	#SUBNET 

	print total_ips
Esempio n. 5
0
def compare_and_update(old_folder,new_folder):
	list=get_list()
	count=0
	cur.execute('USE SENSS')
	new_ip=0
	old_ip=0
	total_add=0
	total_remove=0
	for file_name in list:
		file_to_read=open(str(new_folder)+"/"+file_name,'r')
		print "Doing",file_name
		for line in file_to_read:
			if "#" in line:
				if "Date:" in line:
					data=line.split('Date:')
					time_in_epoch=calendar.timegm(datetime.fromtimestamp(mktime(time.strptime(data[1].strip(),"%a %b  %d %H:%M:%S %Z %Y"))).timetuple())
				continue
		
		with open(str(old_folder)+"/"+file_name) as f:
		    temp=[]
		    for line in f:
		    	if '#' in line:
		    		continue
		    	temp.append(line.strip())
		    t1s = set(temp)
		
		with open(str(new_folder)+'/'+file_name) as f:
		    temp=[]
		    for line in f:
		    	if '#' in line:
		    		continue
		    	temp.append(line.strip())
		    t2s = set(temp)
		
		remove_ip=[]
		add_ip=[]
		for diff in t1s-t2s:
		    remove_ip.append(diff)

		for diff in t2s-t1s:
			add_ip.append(diff)
		#ID 
		#FILE_NAME 
		#IP_ADDRESS 
		#NUMBER_OF_APPEARANCE 
		#START_TIME 
		#DURATION 
		#ACTIVE 
		insert_count=0
		for ip in add_ip:
			cur.execute('SELECT ID FROM IP WHERE IP_ADDRESS=%s and FILE_NAME=%s',(str(ip),str(file_name)))
			if not cur.rowcount:
				insert_count=insert_count+1
				ip_address=line.strip()
				if insert_count%10000==1:
					cmd='INSERT INTO IP(ID,FILE_NAME,IP_ADDRESS,START_TIME,NUMBER_OF_APPEARANCE,ACTIVE)VALUES(NOT NULL,\''+str(file_name)+'\',\''+str(ip_address)+'\',\''+str(time_in_epoch)+'\',1,1)'
				else:
					cmd=cmd+","+'(NOT NULL,\''+str(file_name)+'\',\''+str(ip_address)+'\',\''+str(time_in_epoch)+'\',1,1)'
				if insert_count%10000==0:
					cur.execute(cmd)
					db.commit()
			else:
				for item in cur:
					cur.execute('UPDATE IP SET START_TIME=%s,ACTIVE=1 WHERE ID=%s',(str(time_in_epoch),str(item[0])))
					db.commit()
		if insert_count%10000!=0:
			cur.execute(cmd)
			db.commit()

		for ip in remove_ip:
			cur.execute('SELECT ID,START_TIME,NUMBER_OF_APPEARANCE,DURATION FROM IP WHERE IP_ADDRESS=%s AND FILE_NAME=%s',(str(ip),str(file_name)))
			for item in cur.fetchall():
				id=int(item[0])
				start_time=int(item[1])
				number_of_appearance=int(item[2])
				number_of_appearance=number_of_appearance+1
				end_time=int(calendar.timegm(time.gmtime()))
				duration_string=str(item[3])
				duration=int(end_time)-start_time
				duration_string=duration_string+","+str(duration)
				cur.execute("UPDATE IP SET NUMBER_OF_APPEARANCE=%s,DURATION=%s,ACTIVE=0 WHERE ID=%s",(str(number_of_appearance),str(duration_string),str(id)))
				db.commit()				

	print total_add,total_remove


	return old_ip,new_ip