def __init__(self, spider):
if settings.DEBUG:
loop = asyncio.get_event_loop()
loop.set_debug(enabled=settings.DEBUG)
loop.slow_callback_duration = settings.ASYNC_SLOW_CALLBACK_DURATION
self.spider = spider
self.stats = Stats(controller=self)
self.session = None
self.storage = loader.get_class(settings.STORAGE)(name=self.spider.name, **settings.STORAGE_SETTINGS)
self.throttle = loader.get_class(settings.THROTTLE)(**settings.THROTTLE_SETTINGS)
self.downloader = loader.get_class(settings.DOWNLOADER)(controller=self)
self.middleware = Middlewares(controller=self)
self.pipeline = Pipelines(controller=self)
self.manager = Manager(name=self.spider.name, storage=self.storage)
async def test_receiver():
s = Signal()
data = []
@signals.receiver(signal=s)
async def func(signal, sender, **kwargs):
data.append((signal, sender, kwargs))
classes = [
loader.get_class(c)
for c in (settings.BASE_STARTUP_PIPELINE + settings.STARTUP_PIPELINE)
]
assert len(s.receivers) == 1
assert s.receivers == [weakref.ref(func)]
await StartupPipeline(controller=object()).initialise()
for i, (_signal, _sender, _kwargs) in enumerate(data):
assert isinstance(_signal, signals.Signal)
assert isinstance(_sender, (classes[i], ))
assert _kwargs == {}
with pytest.raises(TypeError) as e:
@signals.receiver(signal=s)
def func(signal, sender, **kwargs):
pass
assert s.receivers == [weakref.ref(func)]
assert "'func' is not coroutine function" in str(e)
def test_get_class():
assert Spider == loader.get_class("okami.Spider")
assert Example == loader.get_class("okami.example.Example")
assert Factory == loader.get_class("tests.factory.Factory")
def pipelines(self):
if not self.cached:
for pipeline in self.sources:
instance = loader.get_class(pipeline)(controller=self.controller)
self.cached.append(instance)
return self.cached
def serve(address=settings.HTTP_SERVER_ADDRESS):
loader.get_class(settings.HTTP_SERVER)(address=address).start()
def middlewares(self):
if not self.cached:
for middleware in self.sources:
self.cached.append(loader.get_class(middleware)(controller=self.controller))
return self.cached