def testBatch(self):
print "start at : %s" % datetime.now()
spider = ImagesDownloader(['http://f1.163.com'])
crawler = GCrawler(spider, workers_count = 8)
crawler.start()
def testDianping(self):
spider = DianpingDownloader(urls)
crawler = GCrawler(spider, workers_count = 3)
crawler.start()
def testDownloader(self):
spider = DummyDownloader(urls)
crawler = GCrawler(spider, workers_count = 3)
crawler.start()
def testSpider(self):
spider = DummySpider()
crawler = GCrawler(spider, workers_count = 3)
crawler.start()
def testBatch(self):
print "start at : %s" % datetime.now()
spider = ImagesDownloader(['http://f1.163.com'])
crawler = GCrawler(spider, workers_count=8)
crawler.start()
def testDownloader(self):
spider = DummyDownloader(urls)
crawler = GCrawler(spider, workers_count=3)
crawler.start()
def testSpider(self):
spider = DummySpider()
crawler = GCrawler(spider, workers_count=3)
crawler.start()
def testDianping(self): spider = DianpingDownloader(urls) crawler = GCrawler(spider, workers_count = 3) crawler.start()