def getRecordUri(self):
req_uri = _URI.fromBytes(self.request.uri)
con_uri = _URI.fromBytes(self.connect_uri)
# Remove default port from URL
if con_uri.port == (80 if con_uri.scheme == 'http' else 443):
con_uri.netloc = con_uri.host
# Copy parameters from the relative req_uri to the con_uri
_copy_attrs(con_uri, req_uri, ['path','params','query','fragment'])
return con_uri.toBytes()
def getRecordUri(request_uri, connect_uri):
"""
:type request_uri: str
:type connect_uri: str
:return: str
"""
req_uri = _URI.fromBytes(request_uri)
con_uri = _URI.fromBytes(connect_uri)
# Remove default port from URL
if con_uri.port == (80 if con_uri.scheme == 'http' else 443):
con_uri.netloc = con_uri.host
# Copy parameters from the relative req_uri to the con_uri
_copy_attrs(con_uri, req_uri, ['path', 'params', 'query', 'fragment'])
return con_uri.toBytes()
def getPage(url, bindAddress = None, *arg, **kw):
# reimplemented here to insert bindAddress
# _parse() deprecated in twisted 13.1.0 in favor of the _URI class
if hasattr(client, '_parse'):
scheme, host, port, path = client._parse(url)
else:
# _URI class renamed to URI in 15.0.0
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
factory = HTTPClientFactory(url, *arg, **kw)
factory.noisy = False
if scheme == 'https':
from twisted.internet import ssl
context = ssl.ClientContextFactory()
reactor.connectSSL(host, port, factory, context,
bindAddress = bindAddress)
else:
reactor.connectTCP(host, port, factory, bindAddress = bindAddress)
return factory.deferred
def getPage(url, bindAddress=None, *arg, **kw):
# reimplemented here to insert bindAddress
# _parse() deprecated in twisted 13.1.0 in favor of the _URI class
if hasattr(client, '_parse'):
scheme, host, port, path = client._parse(url)
else:
# _URI class renamed to URI in 15.0.0
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
factory = HTTPClientFactory(url, *arg, **kw)
factory.noisy = False
if scheme == 'https':
from twisted.internet import ssl
context = ssl.ClientContextFactory()
reactor.connectSSL(host,
port,
factory,
context,
bindAddress=bindAddress)
else:
reactor.connectTCP(host, port, factory, bindAddress=bindAddress)
return factory.deferred
def allHeadersReceived(self):
"""
Parses the HTTP headers and starts a connection to the sever.
After the connection is made, all data should come in raw (body mode)
and should be sent to an HTTPServerParser
"""
self.transport.pauseProducing()
method, request_uri, _ = self.parseHttpStatus(self.status)
self.useSSL = method == 'CONNECT'
connect = reactor.connectTCP
if self.useSSL:
request_uri = 'https://' + request_uri
connect = lambda h,f,p: reactor.connectSSL(h, f, p,
ssl.ClientContextFactory())
if request_uri[:4].lower() != 'http':
# TOFIX: Should check for host in the headers and not just
# the status line
raise ParseError("HTTP status line did not have an absolute uri")
parsedUri = _URI.fromBytes(request_uri)
print "New connection to:", parsedUri.host, parsedUri.port
connect(parsedUri.host, parsedUri.port,
self.clientFactory(self, parsedUri.toBytes()))
HTTPParser.allHeadersReceived(self) # self.switchToBodyMode(None)
def request(self, method, uri, headers=None, bodyProducer=None, follow_redirects=2):
"""
Issue a request to the server indicated by the given C{uri}.
An existing connection from the connection pool may be used or a new one may be created.
I{HTTP} and I{HTTPS} schemes are supported in C{uri}.
@see: L{twisted.web.iweb.IAgent.request}
"""
if uri.startswith('unix://'):
unix_uri = re.match('^unix://(.*)//(.*)', uri)
if not unix_uri:
raise ValueError('Unix pipe http uri format is incorrect.')
filename = '/%s' % unix_uri.group(1)
endpoint = UNIXClientEndpoint(self._reactor, filename)
parsedURI = _URI.fromBytes('unix://unix/%s' % unix_uri.group(2))
parsedURI.host = unix_uri.group(1)
key = (parsedURI.scheme, parsedURI.host, parsedURI.port)
return self._requestWithEndpoint(key, endpoint, method, parsedURI,
headers, bodyProducer,
parsedURI.originForm)
return super(UNIXAwareHttpAgent, self).request(method, uri, headers, bodyProducer)
def massage_url(url):
if url is None or not isinstance(url, basestring):
return None
p = _URI.fromBytes(url)
n = p.netloc.split('.')
n.reverse()
p.scheme = p.netloc = None
return (','.join(n) + ')' + p.toBytes()).lower()
def download_file(url, filename, callback=None, headers=None,
force_filename=False, allow_compression=True):
"""
Downloads a file from a specific URL and returns a Deferred. You can also
specify a callback function to be called as parts are received.
:param url: the url to download from
:type url: string
:param filename: the filename to save the file as
:type filename: string
:param callback: a function to be called when a part of data is received,
it's signature should be: func(data, current_length, total_length)
:type callback: function
:param headers: any optional headers to send
:type headers: dictionary
:param force_filename: force us to use the filename specified rather than
one the server may suggest
:type force_filename: boolean
:param allow_compression: allows gzip & deflate decoding
:type allow_compression: boolean
:returns: the filename of the downloaded file
:rtype: Deferred
:raises t.w.e.PageRedirect: when server responds with a temporary redirect
or permanently moved.
:raises t.w.e.Error: for all other HTTP response errors (besides OK)
"""
url = str(url)
filename = str(filename)
if headers:
for key, value in headers.items():
headers[str(key)] = str(value)
if allow_compression:
if not headers:
headers = {}
headers["accept-encoding"] = "deflate, gzip, x-gzip"
# In twisted 13.1.0 the _parse() function was replaced by the _URI class
if hasattr(client, '_parse'):
scheme, host, port, path = client._parse(url)
else:
from twisted.web.client import _URI
uri = _URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
factory = HTTPDownloader(url, filename, callback, headers, force_filename, allow_compression)
if scheme == "https":
from twisted.internet import ssl
reactor.connectSSL(host, port, factory, ssl.ClientContextFactory())
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs): if hasattr(client, '_parse'): scheme, host, port, path = client._parse(url) else: from twisted.web.client import _URI uri = _URI.fromBytes(url) scheme = uri.scheme host = uri.host port = uri.port path = uri.path self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs) self.connection = reactor.connectTCP(host, port, self.factory)
def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs): if hasattr(client, '_parse'): scheme, host, port, path = client._parse(url) else: from twisted.web.client import _URI uri = _URI.fromBytes(url) scheme = uri.scheme host = uri.host port = uri.port path = uri.path self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs) self.connection = reactor.connectTCP(host, port, self.factory)
def sendUrlCommand(url, contextFactory=None, timeout=60, *args, **kwargs):
if hasattr(client, '_parse'):
scheme, host, port, path = client._parse(url)
else:
from twisted.web.client import _URI
uri = _URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
factory = myHTTPClientFactory(url, *args, **kwargs)
# print "scheme=%s host=%s port=%s path=%s\n" % (scheme, host, port, path)
reactor.connectTCP(host, port, factory, timeout=timeout)
return factory.deferred
def get_raw_page(url, *args, **kwargs):
# In Twisted 13.1.0 _parse() function replaced by _URI class.
# In Twisted 15.0.0 _URI class renamed to URI.
if hasattr(client, "_parse"):
scheme, host, port, path = client._parse(url)
else:
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
factory = client.HTTPClientFactory(url, *args, **kwargs)
reactor.connectTCP(host, port, factory)
return factory
def get_raw_page(url, *args, **kwargs):
# In Twisted 13.1.0 _parse() function replaced by _URI class.
# In Twisted 15.0.0 _URI class renamed to URI.
if hasattr(client, "_parse"):
scheme, host, port, path = client._parse(url)
else:
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
factory = client.HTTPClientFactory(url, *args, **kwargs)
reactor.connectTCP(host, port, factory)
return factory
def getPage(url, contextFactory=None, *args, **kwargs): if hasattr(client, '_parse'): scheme, host, port, path = _parse(url) else: from twisted.web.client import _URI uri = _URI.fromBytes(url) scheme = uri.scheme host = uri.host port = uri.port path = uri.path factory = LimitedHTTPClientFactory(url, *args, **kwargs) if scheme == 'https': from twisted.internet import ssl if contextFactory is None: contextFactory = ssl.ClientContextFactory() reactor.connectSSL(host, port, factory, contextFactory) else: reactor.connectTCP(host, port, factory) return factory.deferred
def __init__(self, url, outputfile, contextFactory = None, *args, **kwargs):
if hasattr(client, '_parse'):
scheme, host, port, path = client._parse(url)
else:
from twisted.web.client import _URI
uri = _URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
if scheme == 'https':
from twisted.internet import ssl
if contextFactory is None:
contextFactory = ssl.ClientContextFactory()
self.connection = reactor.connectSSL(host, port, self.factory, contextFactory)
else:
self.connection = reactor.connectTCP(host, port, self.factory)
return
def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs): if hasattr(client, '_parse'): scheme, host, port, path = client._parse(url) else: # _URI class renamed to URI in 15.0.0 try: from twisted.web.client import _URI as URI except ImportError: from twisted.web.client import URI uri = URI.fromBytes(url) scheme = uri.scheme host = uri.host port = uri.port path = uri.path self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs) if scheme == "https": self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory()) else: self.connection = reactor.connectTCP(host, port, self.factory)
def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
if hasattr(client, "_parse"):
scheme, host, port, path = client._parse(url)
else:
# _URI class renamed to URI in 15.0.0
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
if scheme == "https":
self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
else:
self.connection = reactor.connectTCP(host, port, self.factory)
def sendPartnerBoxWebCommand(url, contextFactory=None, timeout=60, username = "******", password = "", *args, **kwargs):
if hasattr(client, '_parse'):
scheme, host, port, path = _parse(url)
else:
from twisted.web.client import _URI
uri = _URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
basicAuth = encodestring(("%s:%s")%(username,password))
authHeader = "Basic " + basicAuth.strip()
AuthHeaders = {"Authorization": authHeader}
if kwargs.has_key("headers"):
kwargs["headers"].update(AuthHeaders)
else:
kwargs["headers"] = AuthHeaders
factory = myHTTPClientFactory(url, *args, **kwargs)
reactor.connectTCP(host, port, factory, timeout=timeout)
return factory.deferred
def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs): #<<<<<<< HEAD if hasattr(client, '_parse'): scheme, host, port, path = client._parse(url) else: from twisted.web.client import _URI uri = _URI.fromBytes(url) scheme = uri.scheme host = uri.host port = uri.port path = uri.path # ======= another twisted fix possibility # parsed = urlparse(url) # scheme = parsed.scheme # host = parsed.hostname # port = parsed.port or (443 if scheme == 'https' else 80) self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs) if scheme == "https": self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory()) else: self.connection = reactor.connectTCP(host, port, self.factory)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a request to the server indicated by the given C{uri}.
An existing connection from the connection pool may be used or a new one may be created.
Without additional modifications this connection pool may not be very useful because
each connection in the pool will use the same Tor circuit.
Currently only the I{HTTP} scheme is supported in C{uri}.
@see: L{twisted.web.iweb.IAgent.request}
"""
parsedURI = _URI.fromBytes(uri)
endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host,
parsedURI.port)
# XXX
# perhaps the request method should take a key?
key = (parsedURI.scheme, parsedURI.host, parsedURI.port)
return self._requestWithEndpoint(key, endpoint, method, parsedURI,
headers, bodyProducer, parsedURI.originForm)
def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs): #<<<<<<< HEAD if hasattr(client, '_parse'): scheme, host, port, path = client._parse(url) else: from twisted.web.client import _URI uri = _URI.fromBytes(url) scheme = uri.scheme host = uri.host port = uri.port path = uri.path # ======= another twisted fix possibility # parsed = urlparse(url) # scheme = parsed.scheme # host = parsed.hostname # port = parsed.port or (443 if scheme == 'https' else 80) self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs) if scheme == "https": self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory()) else: self.connection = reactor.connectTCP(host, port, self.factory)
def getPageCached(url, contextFactory=None, *args, **kwargs):
"""download a web page as a string, keep a cache of already downloaded pages
Download a page. Return a deferred, which will callback with a
page (as a string) or errback with a description of the error.
See HTTPClientCacheFactory to see what extra args can be passed.
"""
uri = _URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
factory = HTTPClientCacheFactory(url, *args, **kwargs)
if scheme == 'https':
if contextFactory is None:
contextFactory = HTTPSVerifyingContextFactory(host)
reactor.connectSSL(host, port, factory, contextFactory)
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def getPageCached(url, contextFactory=None, *args, **kwargs):
"""download a web page as a string, keep a cache of already downloaded pages
Download a page. Return a deferred, which will callback with a
page (as a string) or errback with a description of the error.
See HTTPClientCacheFactory to see what extra args can be passed.
"""
uri = _URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
factory = HTTPClientCacheFactory(url, *args, **kwargs)
if scheme == 'https':
if contextFactory is None:
contextFactory = HTTPSVerifyingContextFactory(host)
reactor.connectSSL(host, port, factory, contextFactory)
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
if hasattr(client, '_parse'):
scheme, host, port, path = client._parse(url)
else:
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port or (443 if scheme == 'https' else 80)
path = uri.path
self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
if scheme == "https":
from twisted.internet import ssl
if contextFactory is None:
contextFactory = ssl.ClientContextFactory()
self.connection = reactor.connectSSL(host, port, self.factory,
contextFactory)
else:
self.connection = reactor.connectTCP(host, port, self.factory)
def getPageWithHeader(url, contextFactory=None, *args, **kwargs):
"""
Same as twisted.web.client.getPage, but we keep the HTTP header in the
result thanks to the HTTPClientFactoryWithHeader class
"""
if parseAvailable:
scheme, host, port, path = _parse(url)
else:
uri = _URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
factory = HTTPClientFactoryWithHeader(url, *args, **kwargs)
d = factory.deferred
if scheme == 'https':
from twisted.internet import ssl
if contextFactory is None:
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(host, port, factory, contextFactory)
else:
reactor.connectTCP(host, port, factory)
return d
def getPageWithHeader(url, contextFactory=None, *args, **kwargs):
"""
Same as twisted.web.client.getPage, but we keep the HTTP header in the
result thanks to the HTTPClientFactoryWithHeader class
"""
if parseAvailable:
scheme, host, port, path = _parse(url)
else:
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
factory = HTTPClientFactoryWithHeader(url, *args, **kwargs)
d = factory.deferred
if scheme == 'https':
from twisted.internet import ssl
if contextFactory is None:
contextFactory = ssl.ClientContextFactory()
reactor.connectSSL(host, port, factory, contextFactory)
else:
reactor.connectTCP(host, port, factory)
return d
def convertUriToRelative(uri):
""" Converts an absolute URI to a relative one """
parsedURI = _URI.fromBytes(uri)
parsedURI.scheme = parsedURI.netloc = None
return parsedURI.toBytes()
def uriEquals(self, uri, ignoreScheme=False):
self_uri = _URI.fromBytes(self.uri)
comp_uri = _URI.fromBytes(uri)
if ignoreScheme:
self_uri.scheme = comp_uri.scheme = None
return self_uri.toBytes() == comp_uri.toBytes()
def _download_file(url,
filename,
callback=None,
headers=None,
force_filename=False,
allow_compression=True):
"""
Downloads a file from a specific URL and returns a Deferred. A callback
function can be specified to be called as parts are received.
Args:
url (str): The url to download from
filename (str): The filename to save the file as
callback (func): A function to be called when a part of data is received,
it's signature should be: func(data, current_length, total_length)
headers (dict): Any optional headers to send
force_filename (bool): force us to use the filename specified rather than
one the server may suggest
allow_compression (bool): Allows gzip & deflate decoding
Returns:
Deferred: the filename of the downloaded file
Raises:
t.w.e.PageRedirect
t.w.e.Error: for all other HTTP response errors
"""
if allow_compression:
if not headers:
headers = {}
headers['accept-encoding'] = 'deflate, gzip, x-gzip'
url = url.encode('utf8')
filename = filename.encode('utf8')
headers = utf8_encode_structure(headers) if headers else headers
factory = HTTPDownloader(url, filename, callback, headers, force_filename,
allow_compression)
# In Twisted 13.1.0 _parse() function replaced by _URI class.
# In Twisted 15.0.0 _URI class renamed to URI.
if hasattr(client, '_parse'):
scheme, host, port, dummy_path = client._parse(url)
else:
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
finally:
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
if scheme == 'https':
from twisted.internet import ssl
# ClientTLSOptions in Twisted >= 14, see ticket #2765 for details on this addition.
try:
from twisted.internet._sslverify import ClientTLSOptions
except ImportError:
ctx_factory = ssl.ClientContextFactory()
else:
class TLSSNIContextFactory(ssl.ClientContextFactory): # pylint: disable=no-init
"""
A custom context factory to add a server name for TLS connections.
"""
def getContext(self): # NOQA: N802
ctx = ssl.ClientContextFactory.getContext(self)
ClientTLSOptions(host, ctx)
return ctx
ctx_factory = TLSSNIContextFactory()
reactor.connectSSL(host, port, factory, ctx_factory)
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def _download_file(url, filename, callback=None, headers=None, force_filename=False, allow_compression=True):
"""
Downloads a file from a specific URL and returns a Deferred. A callback
function can be specified to be called as parts are received.
Args:
url (str): The url to download from
filename (str): The filename to save the file as
callback (func): A function to be called when a part of data is received,
it's signature should be: func(data, current_length, total_length)
headers (dict): Any optional headers to send
force_filename (bool): force us to use the filename specified rather than
one the server may suggest
allow_compression (bool): Allows gzip & deflate decoding
Returns:
Deferred: the filename of the downloaded file
Raises:
t.w.e.PageRedirect
t.w.e.Error: for all other HTTP response errors
"""
if allow_compression:
if not headers:
headers = {}
headers['accept-encoding'] = 'deflate, gzip, x-gzip'
url = url.encode('utf8')
filename = filename.encode('utf8')
headers = utf8_encode_structure(headers) if headers else headers
factory = HTTPDownloader(url, filename, callback, headers, force_filename, allow_compression)
# In Twisted 13.1.0 _parse() function replaced by _URI class.
# In Twisted 15.0.0 _URI class renamed to URI.
if hasattr(client, '_parse'):
scheme, host, port, dummy_path = client._parse(url)
else:
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
finally:
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
if scheme == 'https':
from twisted.internet import ssl
# ClientTLSOptions in Twisted >= 14, see ticket #2765 for details on this addition.
try:
from twisted.internet._sslverify import ClientTLSOptions
except ImportError:
ctx_factory = ssl.ClientContextFactory()
else:
class TLSSNIContextFactory(ssl.ClientContextFactory): # pylint: disable=no-init
"""
A custom context factory to add a server name for TLS connections.
"""
def getContext(self): # NOQA: N802
ctx = ssl.ClientContextFactory.getContext(self)
ClientTLSOptions(host, ctx)
return ctx
ctx_factory = TLSSNIContextFactory()
reactor.connectSSL(host, port, factory, ctx_factory)
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def download_file(url, filename, callback=None, headers=None, force_filename=False, allow_compression=True):
"""
Downloads a file from a specific URL and returns a Deferred. You can also
specify a callback function to be called as parts are received.
:param url: the url to download from
:type url: string
:param filename: the filename to save the file as
:type filename: string
:param callback: a function to be called when a part of data is received,
it's signature should be: func(data, current_length, total_length)
:type callback: function
:param headers: any optional headers to send
:type headers: dictionary
:param force_filename: force us to use the filename specified rather than
one the server may suggest
:type force_filename: boolean
:param allow_compression: allows gzip & deflate decoding
:type allow_compression: boolean
:returns: the filename of the downloaded file
:rtype: Deferred
:raises t.w.e.PageRedirect: when server responds with a temporary redirect
or permanently moved.
:raises t.w.e.Error: for all other HTTP response errors (besides OK)
"""
url = str(url)
filename = str(filename)
if headers:
for key, value in headers.items():
headers[str(key)] = str(value)
if allow_compression:
if not headers:
headers = {}
headers["accept-encoding"] = "deflate, gzip, x-gzip"
# In Twisted 13.1.0 _parse() function replaced by _URI class.
# In Twisted 15.0.0 _URI class renamed to URI.
if hasattr(client, "_parse"):
scheme, host, port, path = client._parse(url)
else:
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
factory = HTTPDownloader(url, filename, callback, headers, force_filename, allow_compression)
if scheme == "https":
from twisted.internet import ssl
# ClientTLSOptions in Twisted >= 14, see ticket #2765 for details on this addition.
try:
from twisted.internet._sslverify import ClientTLSOptions
except ImportError:
ctx_factory = ssl.ClientContextFactory()
else:
class TLSSNIContextFactory(ssl.ClientContextFactory):
"""
A custom context factory to add a server name for TLS connections.
"""
def getContext(self, hostname=None, port=None):
ctx = ssl.ClientContextFactory.getContext(self)
ClientTLSOptions(host, ctx)
return ctx
ctx_factory = TLSSNIContextFactory()
reactor.connectSSL(host, port, factory, ctx_factory)
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def convertUriToRelative(uri):
""" Converts an absolute URI to a relative one """
parsedURI = _URI.fromBytes(uri)
parsedURI.scheme = parsedURI.netloc = None
return parsedURI.toBytes()
def download_file(url,
filename,
callback=None,
headers=None,
force_filename=False,
allow_compression=True):
"""
Downloads a file from a specific URL and returns a Deferred. You can also
specify a callback function to be called as parts are received.
:param url: the url to download from
:type url: string
:param filename: the filename to save the file as
:type filename: string
:param callback: a function to be called when a part of data is received,
it's signature should be: func(data, current_length, total_length)
:type callback: function
:param headers: any optional headers to send
:type headers: dictionary
:param force_filename: force us to use the filename specified rather than
one the server may suggest
:type force_filename: boolean
:param allow_compression: allows gzip & deflate decoding
:type allow_compression: boolean
:returns: the filename of the downloaded file
:rtype: Deferred
:raises t.w.e.PageRedirect: when server responds with a temporary redirect
or permanently moved.
:raises t.w.e.Error: for all other HTTP response errors (besides OK)
"""
url = str(url)
filename = str(filename)
if headers:
for key, value in headers.items():
headers[str(key)] = str(value)
if allow_compression:
if not headers:
headers = {}
headers["accept-encoding"] = "deflate, gzip, x-gzip"
# In Twisted 13.1.0 _parse() function replaced by _URI class.
# In Twisted 15.0.0 _URI class renamed to URI.
if hasattr(client, "_parse"):
scheme, host, port, path = client._parse(url)
else:
try:
from twisted.web.client import _URI as URI
except ImportError:
from twisted.web.client import URI
uri = URI.fromBytes(url)
scheme = uri.scheme
host = uri.host
port = uri.port
path = uri.path
factory = HTTPDownloader(url, filename, callback, headers, force_filename,
allow_compression)
if scheme == "https":
from twisted.internet import ssl
reactor.connectSSL(host, port, factory, ssl.ClientContextFactory())
else:
reactor.connectTCP(host, port, factory)
return factory.deferred
def uriEquals(self, uri, ignoreScheme=False):
self_uri = _URI.fromBytes(self.uri)
comp_uri = _URI.fromBytes(uri)
if ignoreScheme:
self_uri.scheme = comp_uri.scheme = None
return self_uri.toBytes() == comp_uri.toBytes()
