def test_basic(self): handler = StaticCrawlerHandler() #handler._main() #message = common_settings.mqclient().get("crawler_request", wait_secs = -1) message = { "url" : "http://www.zongheng.com/", "url_class" : None, "root_url" : None, "crawl_priority" : 1, "crawl_depth" : 0, "full_domain" : "zongheng.com", "page_last_modified" : None, "__priority" : 1, "encoding" : None, "encoding_created_time" : None, "redirect_url" : None, } #print message dfd = handler._process(message) print "waiting" def callback(result): print "finished crawling", result dfd.addBoth(callback) reactor.run()
def run(): handler = StaticCrawlerHandler() count = 0 while count < 50: message = common_settings.mqclient().get("crawler_request") if message is not None: handler._process(message) count += 1
def _async_run(): count = {"count" : 0} def callback(result, count): count["count"] += 1 if count["count"] == 50: print "stopped" reactor.stop() handler = StaticCrawlerHandler() for _ in range(50): dfd = handler._main_process_async(None) dfd.addBoth(callback, count) reactor.run()