def test_add_middleware(test_spider, middleware, loop):
manager = spidermw.SpiderMiddlewareManager()
manager._add_middleware(middleware)
assert middleware.process_spider_input in manager.methods[
'process_spider_input']
assert middleware.process_spider_output in manager.methods[
'process_spider_output']
def test_with_process_chain(test_spider, middleware, loop, logger):
request = Request('http://www.url.com', test_spider.parse)
response = Response('http://www.url.com', body=b'')
expected_result = {'dummy': 'result'}
state = {}
manager = spidermw.SpiderMiddlewareManager(*(middleware, ))
result = loop.run_until_complete(
manager.scrape_response(test_spider.parse, response, request, logger,
test_spider))
assert result[0]['dummy'] == expected_result['dummy']
def test_with_process_chain_failure_without_handler(test_spider, loop, logger):
class GenericMiddleware:
def process_spider_input(self, response, spider):
raise Exception('processing failed')
request = Request('http://www.url.com', test_spider.parse)
response = Response('http://www.url.com', body=b'')
expected_exception_value = 'processing failed'
state = {}
manager = spidermw.SpiderMiddlewareManager(*(GenericMiddleware(), ))
exception = loop.run_until_complete(
manager.scrape_response(test_spider.parse, response, request, logger,
test_spider))
assert exception.args[0] == expected_exception_value
def test_with_open_close_spider(test_spider, loop):
class GenericMiddleware:
def __init__(self, state):
self.state = state
def open_spider(self, spider):
self.state['foo'] = 'bar'
def close_spider(self, spider):
self.state.pop('foo')
state = {}
middleware = spidermw.SpiderMiddlewareManager(
*(GenericMiddleware(state), ))
middleware.open_spider(test_spider)
assert state.get('foo') == 'bar'
middleware.close_spider(test_spider)
assert state.get('foo') is None
def test_with_process_chain_failure(test_spider, loop, logger):
class GenericMiddleware:
def process_spider_input(self, response, spider):
raise Exception('processing failed')
def process_spider_exception(self, response, exception, spider):
return 'processing failed'
request = Request('http://www.url.com', test_spider.parse)
response = Response('http://www.url.com', body=b'')
expected_exception_value = 'processing failed'
state = {}
middleware = GenericMiddleware()
manager = spidermw.SpiderMiddlewareManager(*(middleware, ))
assert middleware.process_spider_exception in manager.methods[
'process_spider_exception']
processed_result = loop.run_until_complete(
manager.scrape_response(test_spider.parse, response, request, logger,
test_spider))
assert processed_result == expected_exception_value