def test_request_http_repr(self): r1 = Request('http://www.example.com') self.assertEqual(request_http_repr(r1), 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n') r1 = Request('http://www.example.com/some/page.html?arg=1') self.assertEqual(request_http_repr(r1), 'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n') r1 = Request('http://www.example.com', method='POST', headers={'Content-type': 'text/html'}, body='Some body') self.assertEqual(request_http_repr(r1), 'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
def process_request(self, request): self.stats.inc_value('downloader/request_count') self.stats.inc_value('downloader/request_method_count/%s' % request.method) req_len = len(request_http_repr(request)) self.stats.inc_value('downloader/request_bytes', req_len) return request
def test_request_http_repr(self): r1 = Request('http://www.example.com') self.assertEqual(request_http_repr(r1), 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n') r1 = Request('http://www.example.com/some/page.html?arg=1') self.assertEqual( request_http_repr(r1), 'GET /some/page.html?arg=1 HTTP/1.1\r\nHost: www.example.com\r\n\r\n' ) r1 = Request('http://www.example.com', method='POST', headers={'Content-type': 'text/html'}, body='Some body') self.assertEqual( request_http_repr(r1), 'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body' )