def main():
#logging.Formatter.converter = time.gmtime
#FORMAT = '%(levelname)s %(asctime)-15s %(message)s'
#logging.basicConfig(filename = os.path.join(os.getcwd(),'crawl.log'), level = logger.INFO,format = FORMAT)
SEPARATOR = '/'
if sys.platform == 'win32':
SEPARATOR = "\\"
ZEROACCESS_UDP_PORT = 16471
silent = False
zeroaccess_bootstrap_seeds_path = "Data"+SEPARATOR+"zeroaccess_node_"+str(ZEROACCESS_UDP_PORT)+".dat"
zeroaccess_nodes = ZeroAccessUtil.read_zeroaccess_data_from_file(zeroaccess_bootstrap_seeds_path)
zeroaccess_file_info_path = "Data"+SEPARATOR+"zeroaccess_file.bin"
zeroaccess_file_list = ZeroAccessUtil.read_zeroaccess_file_data_from_bin(zeroaccess_file_info_path)
zeroaccess_protocol = ZeroAccessProtocol()
zeroaccess_protocol.generate_message(zeroaccess_nodes,zeroaccess_file_list)
# get a sample list of zeroaccess nodes map
#ip_list = random.sample(zeroaccess_nodes,16)
#print ip_list
zeroaccess_protocol.set_udp_port(ZEROACCESS_UDP_PORT)
zeroaccess_protocol.set_bootstrap_node_list(zeroaccess_nodes)
t = reactor.listenUDP(ZEROACCESS_UDP_PORT , zeroaccess_protocol)
signal.signal(signal.SIGINT, SIGINT_exit)
#if(crawl_only):
# reactor.callLater(40,zeroaccess_protocol.RestartCrawl)
# reactor.callLater(60,ShutdownGracefully,ZEROACCESS_UDP_PORT,zeroaccess_protocol.get_nodes_map(),zeroaccess_protocol.get_files_map())
newL_query_update_loop = task.LoopingCall(zeroaccess_protocol.RestartCrawl)
newL_query_update_loop.start(2) # call every second
try:
reactor.run()
#except KeyboardInterrupt:
# print "Interrupted by keyboard. Exiting."
# reactor.stop()
except:
print 'Exception caught while interuptting reactor'
pass
def main():
#logging.Formatter.converter = time.gmtime
#FORMAT = '%(levelname)s %(asctime)-15s %(message)s'
#logging.basicConfig(filename = os.path.join(os.getcwd(),'crawl.log'), level = logger.INFO,format = FORMAT)
SEPARATOR = '/'
if sys.platform == 'win32':
SEPARATOR = "\\"
ZEROACCESS_UDP_PORT = 16471
silent = False
zeroaccess_nodes = []
zeroaccess_file_info_path = "Data" + SEPARATOR + "zeroaccess_file.bin"
zeroaccess_file_list = ZeroAccessUtil.read_zeroaccess_file_data_from_bin(
zeroaccess_file_info_path)
zeroaccess_protocol = ZeroAccessProtocol()
interactive = False
crawl_only = False
# 静默状态，监听
# -l
# 使用 getL 查询爬取全网
# -s
# 周期性大量发送 getL 查询，同时积极回应，使用newL推送 ip
# -i
# 做为客户端，发送查询
# -c
try:
opts, args = getopt.getopt(sys.argv[1:], "hp:lis")
except getopt.GetoptError as err:
print str(err)
sys.exit(2)
for o, a in opts:
if o == '-h':
print 'help'
sys.exit()
if o == '-i':
interactive = True
zeroaccess_protocol.set_interactive(interactive)
if o == '-s':
crawl_only = True
if o == '-c':
zeroaccess_nodes = []
ip_path = "Data" + SEPARATOR + "ip_list.txt"
ip_file = open(ip_path)
for ip_line in ip_file:
print ip_line
node = ZeroAccessNode()
ip_int = struct.unpack("I", socket.inet_aton(ip_line))[0]
node.set_ip(ip_int)
zeroaccess_nodes.append(node)
if o == '-l':
silent = True
zeroaccess_protocol.set_silent_state(True)
if o == '-p':
try:
ZEROACCESS_UDP_PORT = int(a)
except ValueError:
print 'Invalid Value'
if ZEROACCESS_UDP_PORT not in [16471, 16470, 16464, 16465]:
port_error_info = 'Crawling Port Not Valid : ' + str(
ZEROACCESS_UDP_PORT)
logger.info(port_error_info)
sys.exit()
port_info = 'Crawling Port : ' + str(ZEROACCESS_UDP_PORT)
logger.info(port_info)
print port_info
# get a sample list of zeroaccess nodes map
#ip_list = random.sample(zeroaccess_nodes,16)
#print ip_list
zeroaccess_bootstrap_seeds_path = "Data" + SEPARATOR + "zeroaccess_node_" + str(
ZEROACCESS_UDP_PORT) + ".dat"
bootstrap_nodes = ZeroAccessUtil.read_zeroaccess_data_from_file(
zeroaccess_bootstrap_seeds_path)
zeroaccess_nodes = zeroaccess_nodes + bootstrap_nodes
zeroaccess_protocol.set_udp_port(ZEROACCESS_UDP_PORT)
zeroaccess_protocol.set_bootstrap_node_list(zeroaccess_nodes)
zeroaccess_protocol.generate_message(zeroaccess_nodes,
zeroaccess_file_list)
t = reactor.listenUDP(ZEROACCESS_UDP_PORT, zeroaccess_protocol)
signal.signal(signal.SIGINT, SIGINT_exit)
if (crawl_only):
reactor.callLater(40, zeroaccess_protocol.RestartCrawl)
reactor.callLater(60, ShutdownGracefully, ZEROACCESS_UDP_PORT,
zeroaccess_protocol.get_nodes_map(),
zeroaccess_protocol.get_files_map())
if (interactive):
newL_query_update_loop = task.LoopingCall(
zeroaccess_protocol.RestartCrawl)
newL_query_update_loop.start(120) # call every second
try:
reactor.run()
#except KeyboardInterrupt:
# print "Interrupted by keyboard. Exiting."
# reactor.stop()
except:
print 'Exception caught while interuptting reactor'
pass
def main():
#logging.Formatter.converter = time.gmtime
#FORMAT = '%(levelname)s %(asctime)-15s %(message)s'
#logging.basicConfig(filename = os.path.join(os.getcwd(),'crawl.log'), level = logger.INFO,format = FORMAT)
SEPARATOR = '/'
if sys.platform == 'win32':
SEPARATOR = "\\"
ZEROACCESS_UDP_PORT = 16471
silent = False
zeroaccess_nodes = []
zeroaccess_file_info_path = "Data"+SEPARATOR+"zeroaccess_file.bin"
zeroaccess_file_list = ZeroAccessUtil.read_zeroaccess_file_data_from_bin(zeroaccess_file_info_path)
zeroaccess_protocol = ZeroAccessProtocol()
interactive = False
crawl_only = False
# 静默状态，监听
# -l
# 使用 getL 查询爬取全网
# -s
# 周期性大量发送 getL 查询，同时积极回应，使用newL推送 ip
# -i
# 做为客户端，发送查询
# -c
try:
opts,args = getopt.getopt(sys.argv[1:],"hp:lis")
except getopt.GetoptError as err:
print str(err)
sys.exit(2)
for o,a in opts:
if o=='-h':
print 'help'
sys.exit()
if o=='-i':
interactive = True
zeroaccess_protocol.set_interactive(interactive)
if o=='-s':
crawl_only = True
if o=='-c':
zeroaccess_nodes = []
ip_path = "Data"+SEPARATOR+"ip_list.txt"
ip_file = open(ip_path)
for ip_line in ip_file:
print ip_line
node = ZeroAccessNode()
ip_int = struct.unpack("I",socket.inet_aton(ip_line))[0]
node.set_ip(ip_int)
zeroaccess_nodes.append(node)
if o=='-l':
silent = True
zeroaccess_protocol.set_silent_state(True)
if o=='-p':
try:
ZEROACCESS_UDP_PORT = int(a)
except ValueError:
print 'Invalid Value'
if ZEROACCESS_UDP_PORT not in [16471,16470,16464,16465]:
port_error_info = 'Crawling Port Not Valid : ' + str(ZEROACCESS_UDP_PORT)
logger.info(port_error_info)
sys.exit()
port_info = 'Crawling Port : ' + str(ZEROACCESS_UDP_PORT)
logger.info(port_info)
print port_info
# get a sample list of zeroaccess nodes map
#ip_list = random.sample(zeroaccess_nodes,16)
#print ip_list
zeroaccess_bootstrap_seeds_path = "Data"+SEPARATOR+"zeroaccess_node_"+str(ZEROACCESS_UDP_PORT)+".dat"
bootstrap_nodes = ZeroAccessUtil.read_zeroaccess_data_from_file(zeroaccess_bootstrap_seeds_path)
zeroaccess_nodes = zeroaccess_nodes + bootstrap_nodes
zeroaccess_protocol.set_udp_port(ZEROACCESS_UDP_PORT)
zeroaccess_protocol.set_bootstrap_node_list(zeroaccess_nodes)
zeroaccess_protocol.generate_message(zeroaccess_nodes,zeroaccess_file_list)
t = reactor.listenUDP(ZEROACCESS_UDP_PORT , zeroaccess_protocol)
signal.signal(signal.SIGINT, SIGINT_exit)
if(crawl_only):
reactor.callLater(40,zeroaccess_protocol.RestartCrawl)
reactor.callLater(60,ShutdownGracefully,ZEROACCESS_UDP_PORT,zeroaccess_protocol.get_nodes_map(),zeroaccess_protocol.get_files_map())
if(interactive):
newL_query_update_loop = task.LoopingCall(zeroaccess_protocol.RestartCrawl)
newL_query_update_loop.start(120) # call every second
try:
reactor.run()
#except KeyboardInterrupt:
# print "Interrupted by keyboard. Exiting."
# reactor.stop()
except:
print 'Exception caught while interuptting reactor'
pass
