Python HttpErrorMiddleware примеры использования

Пример #1

Файл: test_spidermiddleware_httperror.py Проект: zmunk/scrapy

 def setUp(self):
     self.spider = Spider('foo')
     self.mw = HttpErrorMiddleware(
         Settings({'HTTPERROR_ALLOWED_CODES': (402, )}))
     self.req = Request('http://scrapytest.org')
     self.res200, self.res404, self.res402 = _responses(
         self.req, [200, 404, 402])

Пример #2

Файл: test_spidermiddleware_httperror.py Проект: pasdoy/scrapy

class TestHttpErrorMiddlewareSettings(TestCase):
    """Similar test, but with settings"""

    def setUp(self):
        self.spider = Spider("foo")
        self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
        self.req = Request("http://scrapytest.org")
        self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])

    def test_process_spider_input(self):
        self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
        self.assertEquals(None, self.mw.process_spider_input(self.res402, self.spider))

    def test_meta_overrides_settings(self):
        request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})
        res404 = self.res404.copy()
        res404.request = request
        res402 = self.res402.copy()
        res402.request = request

        self.assertEquals(None, self.mw.process_spider_input(res404, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)

    def test_spider_override_settings(self):
        self.spider.handle_httpstatus_list = [404]
        self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)

Пример #3

Файл: test_spidermiddleware_httperror.py Проект: jungan21/Scrapy_Web_Crawling_Framework

class TestHttpErrorMiddleware(TestCase):

    def setUp(self):
        crawler = get_crawler(Spider)
        self.spider = Spider.from_crawler(crawler, name='foo')
        self.mw = HttpErrorMiddleware(Settings({}))
        self.req = Request('http://scrapytest.org')
        self.res200, self.res404 = _responses(self.req, [200, 404])

    def test_process_spider_input(self):
        self.assertEqual(None,
                self.mw.process_spider_input(self.res200, self.spider))
        self.assertRaises(HttpError,
                self.mw.process_spider_input, self.res404, self.spider)

    def test_process_spider_exception(self):
        self.assertEqual([],
                self.mw.process_spider_exception(self.res404,
                        HttpError(self.res404), self.spider))
        self.assertEqual(None,
                self.mw.process_spider_exception(self.res404,
                        Exception(), self.spider))

    def test_handle_httpstatus_list(self):
        res = self.res404.copy()
        res.request = Request('http://scrapytest.org',
                              meta={'handle_httpstatus_list': [404]})
        self.assertEqual(None,
            self.mw.process_spider_input(res, self.spider))

        self.spider.handle_httpstatus_list = [404]
        self.assertEqual(None,
            self.mw.process_spider_input(self.res404, self.spider))

Пример #4

Файл: test_spidermiddleware_httperror.py Проект: jungan21/Scrapy_Web_Crawling_Framework

class TestHttpErrorMiddlewareHandleAll(TestCase):

    def setUp(self):
        self.spider = Spider('foo')
        self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True}))
        self.req = Request('http://scrapytest.org')
        self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])

    def test_process_spider_input(self):
        self.assertEqual(None,
                self.mw.process_spider_input(self.res200, self.spider))
        self.assertEqual(None,
                self.mw.process_spider_input(self.res404, self.spider))

    def test_meta_overrides_settings(self):
        request = Request('http://scrapytest.org',
                              meta={'handle_httpstatus_list': [404]})
        res404 = self.res404.copy()
        res404.request = request
        res402 = self.res402.copy()
        res402.request = request

        self.assertEqual(None,
            self.mw.process_spider_input(res404, self.spider))
        self.assertRaises(HttpError,
                self.mw.process_spider_input, res402, self.spider)

Пример #5

Файл: test_spidermiddleware_httperror.py Проект: shantikumar/Web-Scraper

class TestHttpErrorMiddlewareHandleAll(TestCase):
    def setUp(self):
        self.spider = Spider('foo')
        self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True}))
        self.req = Request('http://scrapytest.org')
        self.res200, self.res404, self.res402 = _responses(
            self.req, [200, 404, 402])

    def test_process_spider_input(self):
        self.assertIsNone(
            self.mw.process_spider_input(self.res200, self.spider))
        self.assertIsNone(
            self.mw.process_spider_input(self.res404, self.spider))

    def test_meta_overrides_settings(self):
        request = Request('http://scrapytest.org',
                          meta={'handle_httpstatus_list': [404]})
        res404 = self.res404.copy()
        res404.request = request
        res402 = self.res402.copy()
        res402.request = request

        self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, res402,
                          self.spider)

    def test_httperror_allow_all_false(self):
        crawler = get_crawler(_HttpErrorSpider)
        mw = HttpErrorMiddleware.from_crawler(crawler)
        request_httpstatus_false = Request(
            'http://scrapytest.org', meta={'handle_httpstatus_all': False})
        request_httpstatus_true = Request('http://scrapytest.org',
                                          meta={'handle_httpstatus_all': True})
        res404 = self.res404.copy()
        res404.request = request_httpstatus_false
        res402 = self.res402.copy()
        res402.request = request_httpstatus_true

        self.assertRaises(HttpError, mw.process_spider_input, res404,
                          self.spider)
        self.assertIsNone(mw.process_spider_input(res402, self.spider))

Пример #6

Файл: test_spidermiddleware_httperror.py Проект: pasdoy/scrapy

class TestHttpErrorMiddleware(TestCase):
    def setUp(self):
        self.spider = Spider("foo")
        self.mw = HttpErrorMiddleware(Settings({}))
        self.req = Request("http://scrapytest.org")
        self.res200, self.res404 = _responses(self.req, [200, 404])

    def test_process_spider_input(self):
        self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)

    def test_process_spider_exception(self):
        self.assertEquals([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
        self.assertEquals(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider))

    def test_handle_httpstatus_list(self):
        res = self.res404.copy()
        res.request = Request("http://scrapytest.org", meta={"handle_httpstatus_list": [404]})
        self.assertEquals(None, self.mw.process_spider_input(res, self.spider))

        self.spider.handle_httpstatus_list = [404]
        self.assertEquals(None, self.mw.process_spider_input(self.res404, self.spider))

Пример #7

Файл: test_spidermiddleware_httperror.py Проект: atharwa-24/scrapy

class TestHttpErrorMiddlewareSettings(TestCase):
    """Similar test, but with settings"""

    def setUp(self):
        self.spider = Spider("foo")
        self.mw = HttpErrorMiddleware(
            Settings({"HTTPERROR_ALLOWED_CODES": (402, )}))
        self.req = Request("http://scrapytest.org")
        self.res200, self.res404, self.res402 = _responses(
            self.req, [200, 404, 402])

    def test_process_spider_input(self):
        self.assertIsNone(
            self.mw.process_spider_input(self.res200, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, self.res404,
                          self.spider)
        self.assertIsNone(
            self.mw.process_spider_input(self.res402, self.spider))

    def test_meta_overrides_settings(self):
        request = Request("http://scrapytest.org",
                          meta={"handle_httpstatus_list": [404]})
        res404 = self.res404.copy()
        res404.request = request
        res402 = self.res402.copy()
        res402.request = request

        self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, res402,
                          self.spider)

    def test_spider_override_settings(self):
        self.spider.handle_httpstatus_list = [404]
        self.assertIsNone(
            self.mw.process_spider_input(self.res404, self.spider))
        self.assertRaises(HttpError, self.mw.process_spider_input, self.res402,
                          self.spider)

Пример #8

Файл: test_spidermiddleware_httperror.py Проект: shantikumar/Web-Scraper

    def test_httperror_allow_all_false(self):
        crawler = get_crawler(_HttpErrorSpider)
        mw = HttpErrorMiddleware.from_crawler(crawler)
        request_httpstatus_false = Request(
            'http://scrapytest.org', meta={'handle_httpstatus_all': False})
        request_httpstatus_true = Request('http://scrapytest.org',
                                          meta={'handle_httpstatus_all': True})
        res404 = self.res404.copy()
        res404.request = request_httpstatus_false
        res402 = self.res402.copy()
        res402.request = request_httpstatus_true

        self.assertRaises(HttpError, mw.process_spider_input, res404,
                          self.spider)
        self.assertIsNone(mw.process_spider_input(res402, self.spider))

Пример #9

Файл: test_spidermiddleware_httperror.py Проект: pasdoy/scrapy

 def setUp(self):
     self.spider = Spider("foo")
     self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
     self.req = Request("http://scrapytest.org")
     self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])

Пример #10

Файл: test_spidermiddleware_httperror.py Проект: pasdoy/scrapy

 def setUp(self):
     self.spider = Spider("foo")
     self.mw = HttpErrorMiddleware(Settings({}))
     self.req = Request("http://scrapytest.org")
     self.res200, self.res404 = _responses(self.req, [200, 404])

Пример #11

Файл: test_spidermiddleware_httperror.py Проект: jungan21/Scrapy_Web_Crawling_Framework

 def setUp(self):
     crawler = get_crawler(Spider)
     self.spider = Spider.from_crawler(crawler, name='foo')
     self.mw = HttpErrorMiddleware(Settings({}))
     self.req = Request('http://scrapytest.org')
     self.res200, self.res404 = _responses(self.req, [200, 404])

Пример #12

Файл: test_spidermiddleware_httperror.py Проект: ArturGaspar/scrapy

 def setUp(self):
     self.spider = Spider('foo')
     self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True}))
     self.req = Request('http://scrapytest.org')
     self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])

Пример #13

Файл: test_spidermiddleware_httperror.py Проект: atharwa-24/scrapy

 def setUp(self):
     self.spider = Spider("foo")
     self.mw = HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
     self.req = Request("http://scrapytest.org")
     self.res200, self.res404, self.res402 = _responses(
         self.req, [200, 404, 402])

Пример #14

 def setUp(self):
     self.spider = Spider('foo')
     self.mw = HttpErrorMiddleware(Settings({}))
     self.req = Request('http://scrapytest.org')
     self.res200, self.res404 = _responses(self.req, [200, 404])