def fire(self,entry_map,interval): self.__interval = interval self.__state = STATE_PROCESSING import id_mgr act_id = id_mgr.get_new_id() id_mgr.set_id("SpiderID",act_id) logginghelper.info(logginghelper.StatisticEventLog(str(act_id),"Spider",str(time.time()),"Start")) import req_gen import net_task for url,processor_class in entry_map: req = req_gen.make_request(url) job = net_task.make_job(req,None,None) cb_obj = processor_class(job) job.execute(cb_obj.handle_response) return True
global _engine if _engine: return False _engine = HTTPClientEngine(site_conn_limit,default_limit) return True def get_engine(): return _engine ######################################################################################## #below coding is just for testing def test_on_response(response): engine = get_engine() print "current request count in pending:"," www.baidu.com:",engine.get_pending_count("www.baidu.com",),"www.qq.com:",engine.get_pending_count("www.qq.com",) print "Connection:",response.headers["Connection"] if __name__ == "__main__": initialize() this_engine = get_engine() import req_gen for i in range(0,30): req = req_gen.make_request("http://www.baidu.com/",no_keep_alive=False) this_engine.fetch(req,test_on_response) req = req_gen.make_request("http://www.qq.com/",no_keep_alive=False) this_engine.fetch(req,test_on_response) print "www.baidu.com:",this_engine.get_pending_count("www.baidu.com",),"www.qq.com:",this_engine.get_pending_count("www.qq.com",) import tornado.ioloop tornado.ioloop.IOLoop.instance().start()