def __init__(self, puller, worker=None, retry=3, debug=False): self._puller = puller self._worker = worker or QueueWorker(redis_conf) if hasattr(puller, 'db'): self._worker.db = puller.db event.connect( self._worker.on_recv, signal=signals.RECV, sender=self._puller) event.connect( self.stop, signal=signals.STOP, sender=self._puller)
def test_crawler(): crawler = Crawler(scrapy_conf) crawler.install() crawler.configure() myspider = Spider(scrapy_conf) event.connect(_resp, signal=signals.RESPONSE, sender=event.Any) crawler.queue.spider_requests.append((myspider, _requests(10))) #crawler.queue.append_spider(myspider) crawler.start() reactor.run()
def __init__(self, settings, puller, worker=None): self.settings = settings self._puller = puller self._crawler = Crawler(settings) self._worker = worker or Worker(redis_conf) self._crawler.install() self._crawler.configure() #can not using event module of ec2 for defer send, fix it! dispatcher.connect( self._on_recv_pull, signal=signals.RECV, sender=self._puller) dispatcher.connect( self._on_err, signal=signals.ERROR, sender=self._puller) event.connect( self._worker.on_recv,signal=signals.RESPONSE, sender=event.Any)
def __init__(self,db,chnl, ws): self._db = db self._chnl = chnl self._pullers = { 'db': ChannelsPuller(db,chnl), 'ws': WsPuller(ws), } #--- recv event.connect( self._on_db_recv, signal=signals.RECV, sender=self._pullers['db'] ) event.connect( self._on_ws_recv, signal=signals.RECV, sender=self._pullers['ws'] ) #-- stop event.connect( self._on_stop, signal=signals.STOP, sender=self._pullers['db'] ) event.connect( self._on_stop, signal=signals.STOP, sender=self._pullers['ws'] ) #--------- self._db.update_table('chnls', self._chnl, { 'pid': self._chnl, #'ping': 60, })
def test_crawler(): crawler = Crawler(scrapy_conf) crawler.install() crawler.configure() myspider = Spider(scrapy_conf) event.connect( _resp, signal=signals.RESPONSE, sender=event.Any) crawler.queue.spider_requests.append( (myspider, _requests(10)) ) #crawler.queue.append_spider(myspider) crawler.start() reactor.run()
def __init__(self, db, chnls, url): self._url = url self._ws = websocket.WebSocket() self._pullers = { 'ws': WsPuller(self._ws,retry=3), 'db': ChannelsPuller(db,chnls), } event.connect( self._on_init, signal=signals.INIT, sender=self._pullers['ws']) event.connect( self._on_stop, signal=signals.STOP, sender=self._pullers['ws']) event.connect( self._on_stop, signal=signals.STOP, sender=self._pullers['db']) event.connect( self._on_ws_recv, signal=signals.RECV, sender=self._pullers['ws'] ) event.connect( self._on_db_recv, signal=signals.RECV, sender=self._pullers['db'] )
def __init__(self, settings, puller, worker=None): self.settings = settings self._puller = puller self._crawler = Crawler(settings) self._worker = worker or Worker(redis_conf) self._crawler.install() self._crawler.configure() #can not using event module of ec2 for defer send, fix it! dispatcher.connect(self._on_recv_pull, signal=signals.RECV, sender=self._puller) dispatcher.connect(self._on_err, signal=signals.ERROR, sender=self._puller) event.connect(self._worker.on_recv, signal=signals.RESPONSE, sender=event.Any)
def __init__(self, url): self._url = url self._ws = websocket.WebSocket() self._puller = WsPuller(self._ws,retry=3) event.connect( self._on_init, signal=signals.INIT, sender=self._puller) event.connect( self._on_recv, signal=signals.RECV, sender=self._puller) event.connect( self._on_stop, signal=signals.STOP, sender=self._puller)
def __init__(self, url): self._url = url self._ws = websocket.WebSocket() self._puller = WsPuller(self._ws, retry=3) event.connect(self._on_init, signal=signals.INIT, sender=self._puller) event.connect(self._on_recv, signal=signals.RECV, sender=self._puller) event.connect(self._on_stop, signal=signals.STOP, sender=self._puller)
def __init__(self, db, chnls, url): self._url = url self._ws = websocket.WebSocket() self._pullers = { 'ws': WsPuller(self._ws, retry=3), 'db': ChannelsPuller(db, chnls), } event.connect(self._on_init, signal=signals.INIT, sender=self._pullers['ws']) event.connect(self._on_stop, signal=signals.STOP, sender=self._pullers['ws']) event.connect(self._on_stop, signal=signals.STOP, sender=self._pullers['db']) event.connect(self._on_ws_recv, signal=signals.RECV, sender=self._pullers['ws']) event.connect(self._on_db_recv, signal=signals.RECV, sender=self._pullers['db'])
def __init__(self, db, chnl, ws): self._db = db self._chnl = chnl self._pullers = {"db": ChannelsPuller(db, chnl), "ws": WsPuller(ws)} # --- recv event.connect(self._on_db_recv, signal=signals.RECV, sender=self._pullers["db"]) event.connect(self._on_ws_recv, signal=signals.RECV, sender=self._pullers["ws"]) # -- stop event.connect(self._on_stop, signal=signals.STOP, sender=self._pullers["db"]) event.connect(self._on_stop, signal=signals.STOP, sender=self._pullers["ws"]) # --------- self._db.update_table( "chnls", self._chnl, { "pid": self._chnl, #'ping': 60, }, )
def fire_stop(self): event.send(signal=signals.STOP,sender=self) def on_init(): pass def on_recv(message): pass @decorator.safe_method() def on_stop(): pass a = A() event.connect( on_init, signal=signals.INIT, sender=a) event.connect( on_recv, signal=signals.RECV, sender=a) event.connect( on_stop, signal=signals.STOP, sender=a) #---------------------------------------- class DecoratorTest(unittest.TestCase): def setUp(self): pass def tearDown(self): pass
def on_init(): pass def on_recv(message): pass @decorator.safe_method() def on_stop(): pass a = A() event.connect(on_init, signal=signals.INIT, sender=a) event.connect(on_recv, signal=signals.RECV, sender=a) event.connect(on_stop, signal=signals.STOP, sender=a) #---------------------------------------- class DecoratorTest(unittest.TestCase): def setUp(self): pass def tearDown(self): pass def testInit(self): a.fire_init() a.fire_recv()