def generate_message(self, zeroaccess_nodes, zeroaccess_file_list):
ret = urllib2.urlopen('https://enabledns.com/ip')
self.local_ip = ret.read()
get_local_ip_info = 'Retrieve local ip : ' + self.local_ip
logger.info(get_local_ip_info)
#faked_ip = self.local_ip
faked_ip = '96.8.117.251'
self.getL_message = ZeroAccessUtil.buildZeroAccessGetLMessage()
self.newL_message = ZeroAccessUtil.buildZeroAccessNewLMessage(faked_ip)
faked_node_info = ZeroAccessNode()
faked_node_info.set_ip(
struct.unpack("I", socket.inet_aton(faked_ip))[0])
faked_node_info.set_time(time.time())
seed_node_size = 15
seed_node_list = random.sample(zeroaccess_nodes, seed_node_size)
seed_node_list.append(faked_node_info)
print 'bootstrap nodes len : ' + str(len(zeroaccess_nodes))
print 'bootstrap file len : ' + str(len(zeroaccess_file_list))
file_list = random.sample(zeroaccess_file_list, 5)
self.retL_message = ZeroAccessUtil.buildZeroAccessretLMessage(
seed_node_list, file_list)
print 'retL message length ' + str(len(self.retL_message))
print 'retL message :\n'
#print ''.join( [ "%02X" % x for x in self.retL_message]).strip()
print self.retL_message.encode('hex')
def main():
query_message = ZeroAccessUtil.buildMessage()
SEPARATOR = '/'
if sys.platform == 'win32':
SEPARATOR = "\\"
message = ZeroAccessUtil.buildMessage()
print message.encode("hex")
nonQueryedNodes = mul.Queue(5000)
zeroaccess_bootstrap_seeds_path = "Data"+SEPARATOR+"zeroaccess_node.dat"
zeroaccess_nodes = ZeroAccessUtil.read_zeroaccess_data_from_file(zeroaccess_bootstrap_seeds_path)
pile = eventlet.GreenPile()
for x in zeroaccess_nodes[:10]:
pile.spawn(query, x,message)
# note that the pile acts as a collection of return values from the functions
# if any exceptions are raised by the function they'll get raised here
key = [ord('2'),ord('p'),ord('t'),ord('f')]
for node, result in zip(zeroaccess_nodes[:10], pile):
if(result == ''):
print 'no response from '+ socket.inet_ntoa(struct.pack('I',node.get_ip()))
continue
print 'received'
original_message = ZeroAccessUtil.xorMessage(result[0],key)
crc32,retL_command,b_flag,ip_count = struct.unpack('IIII',original_message[:16])
print socket.inet_ntoa(struct.pack('I',node.get_ip()))+' --> ip count: '+str(ip_count)
def generate_message(self,zeroaccess_nodes,zeroaccess_file_list):
ret = urllib2.urlopen('https://enabledns.com/ip')
self.local_ip = ret.read()
get_local_ip_info = 'Retrieve local ip : '+self.local_ip
logger.info(get_local_ip_info)
#faked_ip = self.local_ip
faked_ip = '96.8.117.251'
self.getL_message = ZeroAccessUtil.buildZeroAccessGetLMessage()
self.newL_message = ZeroAccessUtil.buildZeroAccessNewLMessage(faked_ip)
faked_node_info = ZeroAccessNode()
faked_node_info.set_ip(struct.unpack("I",socket.inet_aton(faked_ip))[0])
faked_node_info.set_time(time.time())
seed_node_size = 15
seed_node_list = random.sample(zeroaccess_nodes,seed_node_size)
seed_node_list.append(faked_node_info)
print 'bootstrap nodes len : ' + str(len(zeroaccess_nodes))
print 'bootstrap file len : ' + str(len(zeroaccess_file_list))
file_list = random.sample(zeroaccess_file_list,5)
self.retL_message = ZeroAccessUtil.buildZeroAccessretLMessage(seed_node_list,file_list)
print 'retL message length ' + str(len(self.retL_message))
print 'retL message :\n'
#print ''.join( [ "%02X" % x for x in self.retL_message]).strip()
print self.retL_message.encode('hex')
def main():
query_message = ZeroAccessUtil.buildMessage()
SEPARATOR = '/'
if sys.platform == 'win32':
SEPARATOR = "\\"
message = ZeroAccessUtil.buildMessage()
print message.encode("hex")
nonQueryedNodes = mul.Queue(5000)
zeroaccess_bootstrap_seeds_path = "Data" + SEPARATOR + "zeroaccess_node.dat"
zeroaccess_nodes = ZeroAccessUtil.read_zeroaccess_data_from_file(
zeroaccess_bootstrap_seeds_path)
pile = eventlet.GreenPile()
for x in zeroaccess_nodes[:10]:
pile.spawn(query, x, message)
# note that the pile acts as a collection of return values from the functions
# if any exceptions are raised by the function they'll get raised here
key = [ord('2'), ord('p'), ord('t'), ord('f')]
for node, result in zip(zeroaccess_nodes[:10], pile):
if (result == ''):
print 'no response from ' + socket.inet_ntoa(
struct.pack('I', node.get_ip()))
continue
print 'received'
original_message = ZeroAccessUtil.xorMessage(result[0], key)
crc32, retL_command, b_flag, ip_count = struct.unpack(
'IIII', original_message[:16])
print socket.inet_ntoa(struct.pack(
'I', node.get_ip())) + ' --> ip count: ' + str(ip_count)
def main():
#logging.Formatter.converter = time.gmtime
#FORMAT = '%(levelname)s %(asctime)-15s %(message)s'
#logging.basicConfig(filename = os.path.join(os.getcwd(),'crawl.log'), level = logger.INFO,format = FORMAT)
SEPARATOR = '/'
if sys.platform == 'win32':
SEPARATOR = "\\"
ZEROACCESS_UDP_PORT = 16471
silent = False
zeroaccess_bootstrap_seeds_path = "Data"+SEPARATOR+"zeroaccess_node_"+str(ZEROACCESS_UDP_PORT)+".dat"
zeroaccess_nodes = ZeroAccessUtil.read_zeroaccess_data_from_file(zeroaccess_bootstrap_seeds_path)
zeroaccess_file_info_path = "Data"+SEPARATOR+"zeroaccess_file.bin"
zeroaccess_file_list = ZeroAccessUtil.read_zeroaccess_file_data_from_bin(zeroaccess_file_info_path)
zeroaccess_protocol = ZeroAccessProtocol()
zeroaccess_protocol.generate_message(zeroaccess_nodes,zeroaccess_file_list)
# get a sample list of zeroaccess nodes map
#ip_list = random.sample(zeroaccess_nodes,16)
#print ip_list
zeroaccess_protocol.set_udp_port(ZEROACCESS_UDP_PORT)
zeroaccess_protocol.set_bootstrap_node_list(zeroaccess_nodes)
t = reactor.listenUDP(ZEROACCESS_UDP_PORT , zeroaccess_protocol)
signal.signal(signal.SIGINT, SIGINT_exit)
#if(crawl_only):
# reactor.callLater(40,zeroaccess_protocol.RestartCrawl)
# reactor.callLater(60,ShutdownGracefully,ZEROACCESS_UDP_PORT,zeroaccess_protocol.get_nodes_map(),zeroaccess_protocol.get_files_map())
newL_query_update_loop = task.LoopingCall(zeroaccess_protocol.RestartCrawl)
newL_query_update_loop.start(2) # call every second
try:
reactor.run()
#except KeyboardInterrupt:
# print "Interrupted by keyboard. Exiting."
# reactor.stop()
except:
print 'Exception caught while interuptting reactor'
pass
def ShutdownGracefully(udp_port,nodes_map,files_map):
reactor.stop()
info = 'Crawling Eventloop Stopped with node count : ' + str(len(nodes_map))
print info
logger.info(info)
nodes_file_path_prefix = 'log/zeroaccess_nodes_'
files_file_path_prefix = 'log/zeroaccess_nodes_file_'
files_bin_path_prefix = 'log/zeroaccess_nodes_bin_'
ZeroAccessUtil.save_zeroaccess_data_to_csv(nodes_map,nodes_file_path_prefix,udp_port)
ZeroAccessUtil.save_zeroaccess_file_data_to_csv(files_map,files_file_path_prefix,udp_port)
ZeroAccessUtil.save_zeroaccess_file_data_to_bin(files_map,files_bin_path_prefix,udp_port)
def datagramReceived(self, datagram, host):
try:
#print 'host in --> '+str(host[0])
#original_message = ZeroAccessUtil.xorMessage(datagram,self.key)
original_message = ZeroAccessUtil.xorMessageFast(datagram,self.key_int)
crc32,command,b_flag,ip_count = struct.unpack('IIII',original_message[:16])
if(command == self.getL_command_int):
self.getL_process(original_message,host)
elif command == self.retL_command_int:
self.retL_process(original_message,host)
elif command == self.newL_command_int:
self.newL_process(original_message,host)
else:
print 'Unknown command : '+str(hex(command))
except Exception , e:
logger.debug('error in parsing query from node '+str(host))
logger.debug(str(e))
traceback.print_exc()
return
def main():
#logging.Formatter.converter = time.gmtime
#FORMAT = '%(levelname)s %(asctime)-15s %(message)s'
#logging.basicConfig(filename = os.path.join(os.getcwd(),'crawl.log'), level = logger.INFO,format = FORMAT)
SEPARATOR = '/'
if sys.platform == 'win32':
SEPARATOR = "\\"
ZEROACCESS_UDP_PORT = 16471
silent = False
zeroaccess_nodes = []
zeroaccess_file_info_path = "Data" + SEPARATOR + "zeroaccess_file.bin"
zeroaccess_file_list = ZeroAccessUtil.read_zeroaccess_file_data_from_bin(
zeroaccess_file_info_path)
zeroaccess_protocol = ZeroAccessProtocol()
interactive = False
crawl_only = False
# 静默状态，监听
# -l
# 使用 getL 查询爬取全网
# -s
# 周期性大量发送 getL 查询，同时积极回应，使用newL推送 ip
# -i
# 做为客户端，发送查询
# -c
try:
opts, args = getopt.getopt(sys.argv[1:], "hp:lis")
except getopt.GetoptError as err:
print str(err)
sys.exit(2)
for o, a in opts:
if o == '-h':
print 'help'
sys.exit()
if o == '-i':
interactive = True
zeroaccess_protocol.set_interactive(interactive)
if o == '-s':
crawl_only = True
if o == '-c':
zeroaccess_nodes = []
ip_path = "Data" + SEPARATOR + "ip_list.txt"
ip_file = open(ip_path)
for ip_line in ip_file:
print ip_line
node = ZeroAccessNode()
ip_int = struct.unpack("I", socket.inet_aton(ip_line))[0]
node.set_ip(ip_int)
zeroaccess_nodes.append(node)
if o == '-l':
silent = True
zeroaccess_protocol.set_silent_state(True)
if o == '-p':
try:
ZEROACCESS_UDP_PORT = int(a)
except ValueError:
print 'Invalid Value'
if ZEROACCESS_UDP_PORT not in [16471, 16470, 16464, 16465]:
port_error_info = 'Crawling Port Not Valid : ' + str(
ZEROACCESS_UDP_PORT)
logger.info(port_error_info)
sys.exit()
port_info = 'Crawling Port : ' + str(ZEROACCESS_UDP_PORT)
logger.info(port_info)
print port_info
# get a sample list of zeroaccess nodes map
#ip_list = random.sample(zeroaccess_nodes,16)
#print ip_list
zeroaccess_bootstrap_seeds_path = "Data" + SEPARATOR + "zeroaccess_node_" + str(
ZEROACCESS_UDP_PORT) + ".dat"
bootstrap_nodes = ZeroAccessUtil.read_zeroaccess_data_from_file(
zeroaccess_bootstrap_seeds_path)
zeroaccess_nodes = zeroaccess_nodes + bootstrap_nodes
zeroaccess_protocol.set_udp_port(ZEROACCESS_UDP_PORT)
zeroaccess_protocol.set_bootstrap_node_list(zeroaccess_nodes)
zeroaccess_protocol.generate_message(zeroaccess_nodes,
zeroaccess_file_list)
t = reactor.listenUDP(ZEROACCESS_UDP_PORT, zeroaccess_protocol)
signal.signal(signal.SIGINT, SIGINT_exit)
if (crawl_only):
reactor.callLater(40, zeroaccess_protocol.RestartCrawl)
reactor.callLater(60, ShutdownGracefully, ZEROACCESS_UDP_PORT,
zeroaccess_protocol.get_nodes_map(),
zeroaccess_protocol.get_files_map())
if (interactive):
newL_query_update_loop = task.LoopingCall(
zeroaccess_protocol.RestartCrawl)
newL_query_update_loop.start(120) # call every second
try:
reactor.run()
#except KeyboardInterrupt:
# print "Interrupted by keyboard. Exiting."
# reactor.stop()
except:
print 'Exception caught while interuptting reactor'
pass
def main():
#logging.Formatter.converter = time.gmtime
#FORMAT = '%(levelname)s %(asctime)-15s %(message)s'
#logging.basicConfig(filename = os.path.join(os.getcwd(),'crawl.log'), level = logger.INFO,format = FORMAT)
SEPARATOR = '/'
if sys.platform == 'win32':
SEPARATOR = "\\"
ZEROACCESS_UDP_PORT = 16471
silent = False
zeroaccess_nodes = []
zeroaccess_file_info_path = "Data"+SEPARATOR+"zeroaccess_file.bin"
zeroaccess_file_list = ZeroAccessUtil.read_zeroaccess_file_data_from_bin(zeroaccess_file_info_path)
zeroaccess_protocol = ZeroAccessProtocol()
interactive = False
crawl_only = False
# 静默状态，监听
# -l
# 使用 getL 查询爬取全网
# -s
# 周期性大量发送 getL 查询，同时积极回应，使用newL推送 ip
# -i
# 做为客户端，发送查询
# -c
try:
opts,args = getopt.getopt(sys.argv[1:],"hp:lis")
except getopt.GetoptError as err:
print str(err)
sys.exit(2)
for o,a in opts:
if o=='-h':
print 'help'
sys.exit()
if o=='-i':
interactive = True
zeroaccess_protocol.set_interactive(interactive)
if o=='-s':
crawl_only = True
if o=='-c':
zeroaccess_nodes = []
ip_path = "Data"+SEPARATOR+"ip_list.txt"
ip_file = open(ip_path)
for ip_line in ip_file:
print ip_line
node = ZeroAccessNode()
ip_int = struct.unpack("I",socket.inet_aton(ip_line))[0]
node.set_ip(ip_int)
zeroaccess_nodes.append(node)
if o=='-l':
silent = True
zeroaccess_protocol.set_silent_state(True)
if o=='-p':
try:
ZEROACCESS_UDP_PORT = int(a)
except ValueError:
print 'Invalid Value'
if ZEROACCESS_UDP_PORT not in [16471,16470,16464,16465]:
port_error_info = 'Crawling Port Not Valid : ' + str(ZEROACCESS_UDP_PORT)
logger.info(port_error_info)
sys.exit()
port_info = 'Crawling Port : ' + str(ZEROACCESS_UDP_PORT)
logger.info(port_info)
print port_info
# get a sample list of zeroaccess nodes map
#ip_list = random.sample(zeroaccess_nodes,16)
#print ip_list
zeroaccess_bootstrap_seeds_path = "Data"+SEPARATOR+"zeroaccess_node_"+str(ZEROACCESS_UDP_PORT)+".dat"
bootstrap_nodes = ZeroAccessUtil.read_zeroaccess_data_from_file(zeroaccess_bootstrap_seeds_path)
zeroaccess_nodes = zeroaccess_nodes + bootstrap_nodes
zeroaccess_protocol.set_udp_port(ZEROACCESS_UDP_PORT)
zeroaccess_protocol.set_bootstrap_node_list(zeroaccess_nodes)
zeroaccess_protocol.generate_message(zeroaccess_nodes,zeroaccess_file_list)
t = reactor.listenUDP(ZEROACCESS_UDP_PORT , zeroaccess_protocol)
signal.signal(signal.SIGINT, SIGINT_exit)
if(crawl_only):
reactor.callLater(40,zeroaccess_protocol.RestartCrawl)
reactor.callLater(60,ShutdownGracefully,ZEROACCESS_UDP_PORT,zeroaccess_protocol.get_nodes_map(),zeroaccess_protocol.get_files_map())
if(interactive):
newL_query_update_loop = task.LoopingCall(zeroaccess_protocol.RestartCrawl)
newL_query_update_loop.start(120) # call every second
try:
reactor.run()
#except KeyboardInterrupt:
# print "Interrupted by keyboard. Exiting."
# reactor.stop()
except:
print 'Exception caught while interuptting reactor'
pass
