def run(): handler = StaticCrawlerHandler() count = 0 while count < 50: message = common_settings.mqclient().get("crawler_request") if message is not None: handler._process(message) count += 1
def test_basic(self): handler = StaticCrawlerHandler() #handler._main() #message = common_settings.mqclient().get("crawler_request", wait_secs = -1) message = { "url" : "http://www.zongheng.com/", "url_class" : None, "root_url" : None, "crawl_priority" : 1, "crawl_depth" : 0, "full_domain" : "zongheng.com", "page_last_modified" : None, "__priority" : 1, "encoding" : None, "encoding_created_time" : None, "redirect_url" : None, } #print message dfd = handler._process(message) print "waiting" def callback(result): print "finished crawling", result dfd.addBoth(callback) reactor.run()