def test_parse(self):
"""
L{parse} correctly parses a URL into its various components.
"""
# The default port for HTTP is 80.
self.assertEqual(
parse("http://127.0.0.1/"),
("http", "127.0.0.1", 80, "/"))
# The default port for HTTPS is 443.
self.assertEqual(
parse("https://127.0.0.1/"),
("https", "127.0.0.1", 443, "/"))
# Specifying a port.
self.assertEqual(
parse("http://spam:12345/"),
("http", "spam", 12345, "/"))
# Weird (but commonly accepted) structure uses default port.
self.assertEqual(
parse("http://spam:/"),
("http", "spam", 80, "/"))
# Spaces in the hostname are trimmed, the default path is /.
self.assertEqual(
parse("http://foo "),
("http", "foo", 80, "/"))
def _parse_uri(self, uri):
scheme, host, port, path = parse(
str(uri), defaultPort=DEFAULT_PORT)
self.scheme = scheme
self.host = host
self.port = port
self.path = path
def get_page(self, url, *args, **kwds):
"""
Define our own get_page method so that we can easily override the
factory when we need to. This was copied from the following:
* twisted.web.client.getPage
* twisted.web.client._makeGetterFactory
"""
contextFactory = None
scheme, host, port, path = parse(url)
data = kwds.get('postdata', None)
self._method = method = kwds.get('method', 'GET')
self.request_headers = self._headers(kwds.get('headers', {}))
if (self.body_producer is None) and (data is not None):
self.body_producer = FileBodyProducer(StringIO(data))
if self.endpoint.ssl_hostname_verification:
contextFactory = None
else:
contextFactory = WebClientContextFactory()
agent = _get_agent(scheme, host, self.reactor, contextFactory)
if scheme == "https":
self.client.url = url
d = agent.request(method, url, self.request_headers,
self.body_producer)
d.addCallback(self._handle_response)
return d
def _parse_uri(self, uri):
scheme, host, port, path = parse(
str(uri), defaultPort=False)
self.scheme = scheme
self.host = host
self.port = port
self.path = path
def test_externalUnicodeInterference(self):
"""
L{parse} should return C{str} for the scheme, host, and path
elements of its return tuple, even when passed an URL which has
previously been passed to L{urlparse} as a C{unicode} string.
"""
badInput = u"http://example1.com/path"
goodInput = badInput.encode("ascii")
urlparse(badInput)
scheme, host, port, path = parse(goodInput)
self.assertTrue(isinstance(scheme, str))
self.assertTrue(isinstance(host, str))
self.assertTrue(isinstance(path, str))
def get_page(self, url, *args, **kwds):
"""
Define our own get_page method so that we can easily override the
factory when we need to. This was copied from the following:
* twisted.web.client.getPage
* twisted.web.client._makeGetterFactory
"""
contextFactory = None
scheme, host, port, path = parse(url)
self.client = self.factory(url, *args, **kwds)
if scheme == 'https':
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(host, port, self.client, contextFactory)
else:
reactor.connectTCP(host, port, self.client)
return self.client.deferred
def get_page(self, url, *args, **kwds):
"""
Define our own get_page method so that we can easily override the
factory when we need to. This was copied from the following:
* twisted.web.client.getPage
* twisted.web.client._makeGetterFactory
"""
contextFactory = None
scheme, host, port, path = parse(url)
self.client = self.factory(url, *args, **kwds)
if scheme == 'https':
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(host, port, self.client, contextFactory)
else:
reactor.connectTCP(host, port, self.client)
return self.client.deferred
def get_page(self, url, *args, **kwds):
"""
Define our own get_page method so that we can easily override the
factory when we need to. This was copied from the following:
* twisted.web.client.getPage
* twisted.web.client._makeGetterFactory
"""
contextFactory = None
scheme, host, port, path = parse(url)
data = kwds.get('postdata', None)
self._method = method = kwds.get('method', 'GET')
self.request_headers = self._headers(kwds.get('headers', {}))
if (self.body_producer is None) and (data is not None):
self.body_producer = FileBodyProducer(StringIO(data))
if scheme == "https":
proxy_endpoint = os.environ.get("https_proxy")
if proxy_endpoint:
proxy_url = urlparse.urlparse(proxy_endpoint)
endpoint = TCP4ClientEndpoint(self.reactor, proxy_url.hostname, proxy_url.port)
agent = ProxyAgent(endpoint)
else:
if self.endpoint.ssl_hostname_verification:
contextFactory = WebVerifyingContextFactory(host)
else:
contextFactory = WebClientContextFactory()
agent = Agent(self.reactor, contextFactory)
self.client.url = url
d = agent.request(method, url, self.request_headers,
self.body_producer)
else:
proxy_endpoint = os.environ.get("http_proxy")
if proxy_endpoint:
proxy_url = urlparse.urlparse(proxy_endpoint)
endpoint = TCP4ClientEndpoint(self.reactor, proxy_url.hostname, proxy_url.port)
agent = ProxyAgent(endpoint)
else:
agent = Agent(self.reactor)
d = agent.request(method, url, self.request_headers,
self.body_producer)
d.addCallback(self._handle_response)
return d
