def __init__(self, puller, worker=None, retry=3, debug=False):
self._puller = puller
self._worker = worker or QueueWorker(redis_conf)
if hasattr(puller, 'db'):
self._worker.db = puller.db
event.connect( self._worker.on_recv, signal=signals.RECV, sender=self._puller)
event.connect( self.stop, signal=signals.STOP, sender=self._puller)
def test_crawler():
crawler = Crawler(scrapy_conf)
crawler.install()
crawler.configure()
myspider = Spider(scrapy_conf)
event.connect(_resp, signal=signals.RESPONSE, sender=event.Any)
crawler.queue.spider_requests.append((myspider, _requests(10)))
#crawler.queue.append_spider(myspider)
crawler.start()
reactor.run()
def __init__(self, settings, puller, worker=None):
self.settings = settings
self._puller = puller
self._crawler = Crawler(settings)
self._worker = worker or Worker(redis_conf)
self._crawler.install()
self._crawler.configure()
#can not using event module of ec2 for defer send, fix it!
dispatcher.connect( self._on_recv_pull, signal=signals.RECV, sender=self._puller)
dispatcher.connect( self._on_err, signal=signals.ERROR, sender=self._puller)
event.connect( self._worker.on_recv,signal=signals.RESPONSE, sender=event.Any)
def __init__(self,db,chnl, ws):
self._db = db
self._chnl = chnl
self._pullers = {
'db': ChannelsPuller(db,chnl),
'ws': WsPuller(ws),
}
#--- recv
event.connect( self._on_db_recv,
signal=signals.RECV,
sender=self._pullers['db']
)
event.connect( self._on_ws_recv,
signal=signals.RECV,
sender=self._pullers['ws']
)
#-- stop
event.connect( self._on_stop,
signal=signals.STOP,
sender=self._pullers['db']
)
event.connect( self._on_stop,
signal=signals.STOP,
sender=self._pullers['ws']
)
#---------
self._db.update_table('chnls', self._chnl, {
'pid': self._chnl,
#'ping': 60,
})
def test_crawler():
crawler = Crawler(scrapy_conf)
crawler.install()
crawler.configure()
myspider = Spider(scrapy_conf)
event.connect( _resp, signal=signals.RESPONSE, sender=event.Any)
crawler.queue.spider_requests.append( (myspider, _requests(10)) )
#crawler.queue.append_spider(myspider)
crawler.start()
reactor.run()
def __init__(self, db, chnls, url):
self._url = url
self._ws = websocket.WebSocket()
self._pullers = {
'ws': WsPuller(self._ws,retry=3),
'db': ChannelsPuller(db,chnls),
}
event.connect( self._on_init, signal=signals.INIT, sender=self._pullers['ws'])
event.connect( self._on_stop, signal=signals.STOP, sender=self._pullers['ws'])
event.connect( self._on_stop, signal=signals.STOP, sender=self._pullers['db'])
event.connect( self._on_ws_recv,
signal=signals.RECV,
sender=self._pullers['ws']
)
event.connect( self._on_db_recv,
signal=signals.RECV,
sender=self._pullers['db']
)
def __init__(self, settings, puller, worker=None):
self.settings = settings
self._puller = puller
self._crawler = Crawler(settings)
self._worker = worker or Worker(redis_conf)
self._crawler.install()
self._crawler.configure()
#can not using event module of ec2 for defer send, fix it!
dispatcher.connect(self._on_recv_pull,
signal=signals.RECV,
sender=self._puller)
dispatcher.connect(self._on_err,
signal=signals.ERROR,
sender=self._puller)
event.connect(self._worker.on_recv,
signal=signals.RESPONSE,
sender=event.Any)
def __init__(self, url):
self._url = url
self._ws = websocket.WebSocket()
self._puller = WsPuller(self._ws,retry=3)
event.connect( self._on_init, signal=signals.INIT, sender=self._puller)
event.connect( self._on_recv, signal=signals.RECV, sender=self._puller)
event.connect( self._on_stop, signal=signals.STOP, sender=self._puller)
def __init__(self, url):
self._url = url
self._ws = websocket.WebSocket()
self._puller = WsPuller(self._ws, retry=3)
event.connect(self._on_init, signal=signals.INIT, sender=self._puller)
event.connect(self._on_recv, signal=signals.RECV, sender=self._puller)
event.connect(self._on_stop, signal=signals.STOP, sender=self._puller)
def __init__(self, db, chnls, url):
self._url = url
self._ws = websocket.WebSocket()
self._pullers = {
'ws': WsPuller(self._ws, retry=3),
'db': ChannelsPuller(db, chnls),
}
event.connect(self._on_init,
signal=signals.INIT,
sender=self._pullers['ws'])
event.connect(self._on_stop,
signal=signals.STOP,
sender=self._pullers['ws'])
event.connect(self._on_stop,
signal=signals.STOP,
sender=self._pullers['db'])
event.connect(self._on_ws_recv,
signal=signals.RECV,
sender=self._pullers['ws'])
event.connect(self._on_db_recv,
signal=signals.RECV,
sender=self._pullers['db'])
def __init__(self, db, chnl, ws):
self._db = db
self._chnl = chnl
self._pullers = {"db": ChannelsPuller(db, chnl), "ws": WsPuller(ws)}
# --- recv
event.connect(self._on_db_recv, signal=signals.RECV, sender=self._pullers["db"])
event.connect(self._on_ws_recv, signal=signals.RECV, sender=self._pullers["ws"])
# -- stop
event.connect(self._on_stop, signal=signals.STOP, sender=self._pullers["db"])
event.connect(self._on_stop, signal=signals.STOP, sender=self._pullers["ws"])
# ---------
self._db.update_table(
"chnls",
self._chnl,
{
"pid": self._chnl,
#'ping': 60,
},
)
def fire_stop(self):
event.send(signal=signals.STOP,sender=self)
def on_init():
pass
def on_recv(message):
pass
@decorator.safe_method()
def on_stop():
pass
a = A()
event.connect( on_init, signal=signals.INIT, sender=a)
event.connect( on_recv, signal=signals.RECV, sender=a)
event.connect( on_stop, signal=signals.STOP, sender=a)
#----------------------------------------
class DecoratorTest(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def on_init():
pass
def on_recv(message):
pass
@decorator.safe_method()
def on_stop():
pass
a = A()
event.connect(on_init, signal=signals.INIT, sender=a)
event.connect(on_recv, signal=signals.RECV, sender=a)
event.connect(on_stop, signal=signals.STOP, sender=a)
#----------------------------------------
class DecoratorTest(unittest.TestCase):
def setUp(self):
pass
def tearDown(self):
pass
def testInit(self):
a.fire_init()
a.fire_recv()
