def _clientfactory(url, *args, **kwargs): url = to_unicode(url) timeout = kwargs.pop('timeout', 0) f = client.ScrapyHTTPClientFactory(Request(url, *args, **kwargs), timeout=timeout) f.deferred.addCallback(response_transform or (lambda r: r.body)) return f
def test_non_standard_line_endings(self): # regression test for: http://dev.scrapy.org/ticket/258 factory = client.ScrapyHTTPClientFactory(Request( url='http://foo/bar')) protocol = client.ScrapyHTTPPageGetter() protocol.factory = factory protocol.headers = Headers() protocol.dataReceived(b"HTTP/1.0 200 OK\n") protocol.dataReceived(b"Hello: World\n") protocol.dataReceived(b"Foo: Bar\n") protocol.dataReceived(b"\n") self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']}))
def test_earlyHeaders(self): # basic test stolen from twisted HTTPageGetter factory = client.ScrapyHTTPClientFactory( Request(url='http://foo/bar', body="some data", headers={ 'Host': 'example.net', 'User-Agent': 'fooble', 'Cookie': 'blah blah', 'Content-Length': '12981', 'Useful': 'value' })) self._test( factory, b"GET /bar HTTP/1.0\r\n" b"Content-Length: 9\r\n" b"Useful: value\r\n" b"Connection: close\r\n" b"User-Agent: fooble\r\n" b"Host: example.net\r\n" b"Cookie: blah blah\r\n" b"\r\n" b"some data") # test minimal sent headers factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar')) self._test(factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n") # test a simple POST with body and content-type factory = client.ScrapyHTTPClientFactory( Request( method='POST', url='http://foo/bar', body='name=value', headers={'Content-Type': 'application/x-www-form-urlencoded'})) self._test( factory, b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Connection: close\r\n" b"Content-Type: application/x-www-form-urlencoded\r\n" b"Content-Length: 10\r\n" b"\r\n" b"name=value") # test a POST method with no body provided factory = client.ScrapyHTTPClientFactory( Request(method='POST', url='http://foo/bar')) self._test( factory, b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" b"\r\n") # test with single and multivalued headers factory = client.ScrapyHTTPClientFactory( Request(url='http://foo/bar', headers={ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], })) self._test( factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"X-Meta-Multivalued: value1\r\n" b"X-Meta-Multivalued: value2\r\n" b"X-Meta-Single: single\r\n" b"\r\n") # same test with single and multivalued headers but using Headers class factory = client.ScrapyHTTPClientFactory( Request(url='http://foo/bar', headers=Headers({ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], }))) self._test( factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"X-Meta-Multivalued: value1\r\n" b"X-Meta-Multivalued: value2\r\n" b"X-Meta-Single: single\r\n" b"\r\n")
def _parse(self, url): f = client.ScrapyHTTPClientFactory(Request(url)) return (f.scheme, f.netloc, f.host, f.port, f.path)
def testFactoryInfo(self): url = self.getURL('file') _, _, host, port, _ = client._parse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) reactor.connectTCP(to_unicode(host), port, factory) return factory.deferred.addCallback(self._cbFactoryInfo, factory)
def testFactoryInfo(self): url = self.getURL('file') scheme, netloc, host, port, path = client._parse(url) factory = client.ScrapyHTTPClientFactory(Request(url)) reactor.connectTCP(host, port, factory) return factory.deferred.addCallback(self._cbFactoryInfo, factory)
def _clientfactory(*args, **kwargs): timeout = kwargs.pop('timeout', 0) f = client.ScrapyHTTPClientFactory(Request(*args, **kwargs), timeout=timeout) f.deferred.addCallback(lambda r: r.body) return f