Python request_http_repr Examples

Example #1

File: test_utils_request.py Project: Mimino666/crawlmi

    def test_request_http_repr(self):
        r1 = Request('http://www.example.com')
        self.assertEqual(request_http_repr(r1), 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')

        r1 = Request('http://www.example.com/some/page.html?arg=1')
        self.assertEqual(request_http_repr(r1), 'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n')

        r1 = Request('http://www.example.com', method='POST', headers={'Content-type': 'text/html'}, body='Some body')
        self.assertEqual(request_http_repr(r1), 'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')

Example #2

File: downloader_stats.py Project: dahuangfeng123/crawlmi

 def process_request(self, request):
     self.stats.inc_value('downloader/request_count')
     self.stats.inc_value('downloader/request_method_count/%s' %
                          request.method)
     req_len = len(request_http_repr(request))
     self.stats.inc_value('downloader/request_bytes', req_len)
     return request

Example #3

File: test_utils_request.py Project: dahuangfeng123/crawlmi

    def test_request_http_repr(self):
        r1 = Request('http://www.example.com')
        self.assertEqual(request_http_repr(r1),
                         'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n')

        r1 = Request('http://www.example.com/some/page.html?arg=1')
        self.assertEqual(
            request_http_repr(r1),
            'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n'
        )

        r1 = Request('http://www.example.com',
                     method='POST',
                     headers={'Content-type': 'text/html'},
                     body='Some body')
        self.assertEqual(
            request_http_repr(r1),
            'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body'
        )

Example #4

File: downloader_stats.py Project: Mimino666/crawlmi

 def process_request(self, request):
     self.stats.inc_value('downloader/request_count')
     self.stats.inc_value('downloader/request_method_count/%s' % request.method)
     req_len = len(request_http_repr(request))
     self.stats.inc_value('downloader/request_bytes', req_len)
     return request