def https_open(self, req):
# type: (Request) -> addinfourl
"""Return an addinfourl object for the request, using http_class.
http_class must implement the HTTPConnection API from httplib.
The addinfourl return value is a file-like object. It also
has methods and attributes including:
- info(): return a mimetools.Message object for the headers
- geturl(): return the original request URL
- code: HTTP status code
"""
# https://docs.python.org/3.3/library/urllib.request.html#urllib.request.Request.get_host
try: # up to python-3.2
host = req.get_host()
except AttributeError: # from python-3.3
host = req.host
if not host:
raise URLError('no host given')
# Our change: Check to see if we're using a proxy.
# Then create an appropriate ssl-aware connection.
full_url = req.get_full_url()
target_host = url_parse(full_url)[1]
if target_host != host:
request_uri = urldefrag(full_url)[0]
h = httpslib.ProxyHTTPSConnection(host=host, ssl_context=self.ctx)
else:
try: # up to python-3.2
request_uri = req.get_selector()
except AttributeError: # from python-3.3
request_uri = req.selector
h = httpslib.HTTPSConnection(host=host, ssl_context=self.ctx)
# End our change
h.set_debuglevel(self._debuglevel)
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
# We want to make an HTTP/1.1 request, but the addinfourl
# class isn't prepared to deal with a persistent connection.
# It will try to read all remaining data from the socket,
# which will block while the server waits for the next request.
# So make sure the connection gets closed after the (only)
# request.
headers["Connection"] = "close"
try:
h.request(req.get_method(), request_uri, req.data, headers)
r = h.getresponse()
except socket.error as err: # XXX what error?
raise URLError(err)
# Pick apart the HTTPResponse object to get the addinfourl
# object initialized properly.
# Wrap the HTTPResponse object in socket's file object adapter
# for Windows. That adapter calls recv(), so delegate recv()
# to read(). This weird wrapping allows the returned object to
# have readline() and readlines() methods.
r.recv = r.read
if six.PY2:
fp = socket._fileobject(r, close=True)
else:
r._decref_socketios = lambda: None
r.ssl = h.sock.ssl
r._timeout = -1.0
r.recv_into = r.readinto
fp = socket.SocketIO(r, 'rb')
resp = addinfourl(fp, r.msg, req.get_full_url())
resp.code = r.status
resp.msg = r.reason
return resp
def https_open(self, req):
# type: (Request) -> addinfourl
"""Return an addinfourl object for the request, using http_class.
http_class must implement the HTTPConnection API from httplib.
The addinfourl return value is a file-like object. It also
has methods and attributes including:
- info(): return a mimetools.Message object for the headers
- geturl(): return the original request URL
- code: HTTP status code
"""
# https://docs.python.org/3.3/library/urllib.request.html#urllib.request.Request.get_host
try: # up to python-3.2
host = req.get_host()
except AttributeError: # from python-3.3
host = req.host
if not host:
raise URLError('no host given')
# Our change: Check to see if we're using a proxy.
# Then create an appropriate ssl-aware connection.
full_url = req.get_full_url()
target_host = url_parse(full_url)[1]
if target_host != host:
request_uri = urldefrag(full_url)[0]
h = httpslib.ProxyHTTPSConnection(host=host, ssl_context=self.ctx)
else:
try: # up to python-3.2
request_uri = req.get_selector()
except AttributeError: # from python-3.3
request_uri = req.selector
h = httpslib.HTTPSConnection(host=host, ssl_context=self.ctx)
# End our change
h.set_debuglevel(self._debuglevel)
headers = dict(req.headers)
headers.update(req.unredirected_hdrs)
# We want to make an HTTP/1.1 request, but the addinfourl
# class isn't prepared to deal with a persistent connection.
# It will try to read all remaining data from the socket,
# which will block while the server waits for the next request.
# So make sure the connection gets closed after the (only)
# request.
headers["Connection"] = "close"
try:
h.request(req.get_method(), request_uri, req.data, headers)
r = h.getresponse()
except socket.error as err: # XXX what error?
raise URLError(err)
# Pick apart the HTTPResponse object to get the addinfourl
# object initialized properly.
# Wrap the HTTPResponse object in socket's file object adapter
# for Windows. That adapter calls recv(), so delegate recv()
# to read(). This weird wrapping allows the returned object to
# have readline() and readlines() methods.
r.recv = r.read
if six.PY2:
fp = socket._fileobject(r, close=True)
else:
r._decref_socketios = lambda: None
r.ssl = h.sock.ssl
r._timeout = -1.0
r.recv_into = lambda b: SSL.Connection.recv_into(r, b)
fp = socket.SocketIO(r, 'rb')
resp = addinfourl(fp, r.msg, req.get_full_url())
resp.code = r.status
resp.msg = r.reason
return resp
def open_https(self, url, data=None, ssl_context=None):
# type: (URLOpener, AnyStr, Optional[bytes], Optional[SSL.Context]) -> addinfourl
"""
Open URL over the SSL connection.
:param url: URL to be opened
:param data: data for the POST request
:param ssl_context: SSL.Context to be used
:return:
"""
if six.PY3:
warnings.warn('URLOpener has been deprecated in Py3k',
DeprecationWarning)
if ssl_context is not None and isinstance(ssl_context, SSL.Context):
self.ctx = ssl_context
else:
self.ctx = SSL.Context()
user_passwd = None
if isinstance(url, six.string_types):
try: # python 2
# http://pydoc.org/2.5.1/urllib.html
host, selector = splithost(url)
if host:
user_passwd, host = splituser(host)
host = unquote(host)
realhost = host
except NameError: # python 3 has no splithost
# https://docs.python.org/3/library/urllib.parse.html
parsed = urlparse(url)
host = parsed.hostname
if parsed.port:
host += ":{0}".format(parsed.port)
user_passwd = parsed.password
if parsed.password:
user_passwd += ":{0}".format(parsed.password)
selector = parsed.path
else:
host, selector = url
urltype, rest = splittype(selector)
url = rest
user_passwd = None
if urltype.lower() != 'http':
realhost = None
else:
try: # python 2
realhost, rest = splithost(rest)
if realhost:
user_passwd, realhost = splituser(realhost)
if user_passwd:
selector = "%s://%s%s" % (urltype, realhost, rest)
except NameError: # python 3 has no splithost
parsed = urlparse(rest)
host = parsed.hostname
if parsed.port:
host += ":{0}".format(parsed.port)
user_passwd = parsed.username
if parsed.password:
user_passwd += ":{0}".format(parsed.password)
# print("proxy via http:", host, selector)
if not host:
raise IOError('http error', 'no host given')
if user_passwd:
if six.PY3:
auth = base64.encodebytes(user_passwd).strip()
else:
auth = base64.encodestring(user_passwd).strip()
else:
auth = None
# Start here!
h = httpslib.HTTPSConnection(host=host, ssl_context=self.ctx)
# h.set_debuglevel(1)
# Stop here!
if data is not None:
h.putrequest('POST', selector)
h.putheader('Content-type', 'application/x-www-form-urlencoded')
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest('GET', selector)
if auth:
h.putheader('Authorization', 'Basic %s' % auth)
for args in self.addheaders:
h.putheader(*args) # for python3 - used to use apply
h.endheaders()
if data is not None:
h.send(data + '\r\n')
# Here again!
resp = h.getresponse()
fp = resp.fp
return addinfourl(fp, resp.msg, "https:" + url)
def open_https(self, url, data=None, ssl_context=None):
# type: (URLOpener, AnyStr, Optional[bytes], Optional[SSL.Context]) -> addinfourl
"""
Open URL over the SSL connection.
:param url: URL to be opened
:param data: data for the POST request
:param ssl_context: SSL.Context to be used
:return:
"""
if six.PY3:
warnings.warn('URLOpener has been deprecated in Py3k', DeprecationWarning)
if ssl_context is not None and isinstance(ssl_context, SSL.Context):
self.ctx = ssl_context
else:
self.ctx = SSL.Context()
user_passwd = None
if isinstance(url, six.string_types):
try: # python 2
# http://pydoc.org/2.5.1/urllib.html
host, selector = splithost(url)
if host:
user_passwd, host = splituser(host)
host = unquote(host)
realhost = host
except NameError: # python 3 has no splithost
# https://docs.python.org/3/library/urllib.parse.html
parsed = urlparse(url)
host = parsed.hostname
if parsed.port:
host += ":{0}".format(parsed.port)
user_passwd = parsed.password
if parsed.password:
user_passwd += ":{0}".format(parsed.password)
selector = parsed.path
else:
host, selector = url
urltype, rest = splittype(selector)
url = rest
user_passwd = None
if urltype.lower() != 'http':
realhost = None
else:
try: # python 2
realhost, rest = splithost(rest)
if realhost:
user_passwd, realhost = splituser(realhost)
if user_passwd:
selector = "%s://%s%s" % (urltype, realhost, rest)
except NameError: # python 3 has no splithost
parsed = urlparse(rest)
host = parsed.hostname
if parsed.port:
host += ":{0}".format(parsed.port)
user_passwd = parsed.username
if parsed.password:
user_passwd += ":{0}".format(parsed.password)
# print("proxy via http:", host, selector)
if not host:
raise IOError('http error', 'no host given')
if user_passwd:
if six.PY3:
auth = base64.encodebytes(user_passwd).strip()
else:
auth = base64.encodestring(user_passwd).strip()
else:
auth = None
# Start here!
h = httpslib.HTTPSConnection(host=host, ssl_context=self.ctx)
# h.set_debuglevel(1)
# Stop here!
if data is not None:
h.putrequest('POST', selector)
h.putheader('Content-type', 'application/x-www-form-urlencoded')
h.putheader('Content-length', '%d' % len(data))
else:
h.putrequest('GET', selector)
if auth:
h.putheader('Authorization', 'Basic %s' % auth)
for args in self.addheaders:
h.putheader(*args) # for python3 - used to use apply
h.endheaders()
if data is not None:
h.send(data + '\r\n')
# Here again!
resp = h.getresponse()
fp = resp.fp
return addinfourl(fp, resp.msg, "https:" + url)