def setUp(self):
"""set up"""
config = {}
mini_spider.read_conf("spider.conf", config)
thread_safe_set = safe_utils.thread_safe(set)
urlSet = thread_safe_set()
queue = Queue.Queue(maxsize=-1)
rlock = threading.RLock()
self.spider = spider.MiniSpider(config, queue, urlSet, rlock)
t = spider.MiniSpider(config, queue, urlSet, rlock)
threads.append(t)
else:
error = "config threadcount is not a positive integer,\
please check your config file, exiting ..."
log.log("error", error)
sys.exit(1)
for i in range(len(threads)):
log.log("info", threads[i].getName() + " is starting")
threads[i].setDaemon(True)
threads[i].start()
print "now spidering ,please wait ~~~"
queue.join()
info = "now all done , and exiting"
log.log("info", info)
print "spider done ~~"
else:
error = "config read failed!"
log.log("error", error)
sys.exit(1)
if __name__ == '__main__':
config = {}
#Bloom Filters
thread_safe_set = safe_utils.thread_safe(set)
urlSet = thread_safe_set()
queue = Queue.Queue(maxsize = -1)
rlock = threading.RLock()
main(config, queue, urlSet, rlock)