def setUp(self): self.spider = Spider('foo') self.mw = HttpErrorMiddleware( Settings({'HTTPERROR_ALLOWED_CODES': (402, )})) self.req = Request('http://scrapytest.org') self.res200, self.res404, self.res402 = _responses( self.req, [200, 404, 402])
class TestHttpErrorMiddlewareSettings(TestCase): """Similar test, but with settings""" def setUp(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) self.assertEquals(None, self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() res402.request = request self.assertEquals(None, self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)
class TestHttpErrorMiddleware(TestCase): def setUp(self): crawler = get_crawler(Spider) self.spider = Spider.from_crawler(crawler, name='foo') self.mw = HttpErrorMiddleware(Settings({})) self.req = Request('http://scrapytest.org') self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) def test_process_spider_exception(self): self.assertEqual([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) self.assertEqual(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider)) def test_handle_httpstatus_list(self): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) self.assertEqual(None, self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider))
class TestHttpErrorMiddlewareHandleAll(TestCase): def setUp(self): self.spider = Spider('foo') self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True})) self.req = Request('http://scrapytest.org') self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() res402.request = request self.assertEqual(None, self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
class TestHttpErrorMiddlewareHandleAll(TestCase): def setUp(self): self.spider = Spider('foo') self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True})) self.req = Request('http://scrapytest.org') self.res200, self.res404, self.res402 = _responses( self.req, [200, 404, 402]) def test_process_spider_input(self): self.assertIsNone( self.mw.process_spider_input(self.res200, self.spider)) self.assertIsNone( self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() res402.request = request self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_httperror_allow_all_false(self): crawler = get_crawler(_HttpErrorSpider) mw = HttpErrorMiddleware.from_crawler(crawler) request_httpstatus_false = Request( 'http://scrapytest.org', meta={'handle_httpstatus_all': False}) request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True}) res404 = self.res404.copy() res404.request = request_httpstatus_false res402 = self.res402.copy() res402.request = request_httpstatus_true self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) self.assertIsNone(mw.process_spider_input(res402, self.spider))
class TestHttpErrorMiddleware(TestCase): def setUp(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({})) self.req = Request("http://scrapytest.org") self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) def test_process_spider_exception(self): self.assertEquals([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) self.assertEquals(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider)) def test_handle_httpstatus_list(self): res = self.res404.copy() res.request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]}) self.assertEquals(None, self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))
class TestHttpErrorMiddlewareSettings(TestCase): """Similar test, but with settings""" def setUp(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware( Settings({"HTTPERROR_ALLOWED_CODES": (402, )})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses( self.req, [200, 404, 402]) def test_process_spider_input(self): self.assertIsNone( self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) self.assertIsNone( self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]}) res404 = self.res404.copy() res404.request = request res402 = self.res402.copy() res402.request = request self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] self.assertIsNone( self.mw.process_spider_input(self.res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)
def test_httperror_allow_all_false(self): crawler = get_crawler(_HttpErrorSpider) mw = HttpErrorMiddleware.from_crawler(crawler) request_httpstatus_false = Request( 'http://scrapytest.org', meta={'handle_httpstatus_all': False}) request_httpstatus_true = Request('http://scrapytest.org', meta={'handle_httpstatus_all': True}) res404 = self.res404.copy() res404.request = request_httpstatus_false res402 = self.res402.copy() res402.request = request_httpstatus_true self.assertRaises(HttpError, mw.process_spider_input, res404, self.spider) self.assertIsNone(mw.process_spider_input(res402, self.spider))
def setUp(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def setUp(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({})) self.req = Request("http://scrapytest.org") self.res200, self.res404 = _responses(self.req, [200, 404])
def setUp(self): crawler = get_crawler(Spider) self.spider = Spider.from_crawler(crawler, name='foo') self.mw = HttpErrorMiddleware(Settings({})) self.req = Request('http://scrapytest.org') self.res200, self.res404 = _responses(self.req, [200, 404])
def setUp(self): self.spider = Spider('foo') self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True})) self.req = Request('http://scrapytest.org') self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def setUp(self): self.spider = Spider("foo") self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True})) self.req = Request("http://scrapytest.org") self.res200, self.res404, self.res402 = _responses( self.req, [200, 404, 402])
def setUp(self): self.spider = Spider('foo') self.mw = HttpErrorMiddleware(Settings({})) self.req = Request('http://scrapytest.org') self.res200, self.res404 = _responses(self.req, [200, 404])