def getRecordUri(self):
     req_uri = _URI.fromBytes(self.request.uri)
     con_uri = _URI.fromBytes(self.connect_uri)
     # Remove default port from URL
     if con_uri.port == (80 if con_uri.scheme == 'http' else 443):
         con_uri.netloc = con_uri.host
     # Copy parameters from the relative req_uri to the con_uri
     _copy_attrs(con_uri, req_uri, ['path','params','query','fragment'])
     return con_uri.toBytes()
Example #2
0
 def getRecordUri(request_uri, connect_uri):
     """
     :type request_uri: str
     :type connect_uri: str
     :return: str
     """
     req_uri = _URI.fromBytes(request_uri)
     con_uri = _URI.fromBytes(connect_uri)
     # Remove default port from URL
     if con_uri.port == (80 if con_uri.scheme == 'http' else 443):
         con_uri.netloc = con_uri.host
     # Copy parameters from the relative req_uri to the con_uri
     _copy_attrs(con_uri, req_uri, ['path', 'params', 'query', 'fragment'])
     return con_uri.toBytes()
Example #3
0
def getPage(url, bindAddress = None, *arg, **kw):
    # reimplemented here to insert bindAddress

    # _parse() deprecated in twisted 13.1.0 in favor of the _URI class
    if hasattr(client, '_parse'):
        scheme, host, port, path = client._parse(url)
    else:
        # _URI class renamed to URI in 15.0.0
        try:
            from twisted.web.client import _URI as URI
        except ImportError:
            from twisted.web.client import URI
        
        uri = URI.fromBytes(url)
        scheme = uri.scheme
        host = uri.host
        port = uri.port
        path = uri.path

    factory = HTTPClientFactory(url, *arg, **kw)
    factory.noisy = False
	
    if scheme == 'https':
        from twisted.internet import ssl
        context = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, context, 
            bindAddress = bindAddress)
    else:
        reactor.connectTCP(host, port, factory, bindAddress = bindAddress)
    return factory.deferred
Example #4
0
def getPage(url, bindAddress=None, *arg, **kw):
    # reimplemented here to insert bindAddress

    # _parse() deprecated in twisted 13.1.0 in favor of the _URI class
    if hasattr(client, '_parse'):
        scheme, host, port, path = client._parse(url)
    else:
        # _URI class renamed to URI in 15.0.0
        try:
            from twisted.web.client import _URI as URI
        except ImportError:
            from twisted.web.client import URI

        uri = URI.fromBytes(url)
        scheme = uri.scheme
        host = uri.host
        port = uri.port
        path = uri.path

    factory = HTTPClientFactory(url, *arg, **kw)
    factory.noisy = False

    if scheme == 'https':
        from twisted.internet import ssl
        context = ssl.ClientContextFactory()
        reactor.connectSSL(host,
                           port,
                           factory,
                           context,
                           bindAddress=bindAddress)
    else:
        reactor.connectTCP(host, port, factory, bindAddress=bindAddress)
    return factory.deferred
 def allHeadersReceived(self):
     """
     Parses the HTTP headers and starts a connection to the sever.
     After the connection is made, all data should come in raw (body mode)
     and should be sent to an HTTPServerParser
     """
     self.transport.pauseProducing()
     method, request_uri, _ = self.parseHttpStatus(self.status)
     
     self.useSSL = method == 'CONNECT'
     connect = reactor.connectTCP
     if self.useSSL:
         request_uri = 'https://' + request_uri
         connect = lambda h,f,p: reactor.connectSSL(h, f, p,
                                                  ssl.ClientContextFactory())
     if request_uri[:4].lower() != 'http':
         # TOFIX: Should check for host in the headers and not just
         # the status line
         raise ParseError("HTTP status line did not have an absolute uri")
     
     parsedUri = _URI.fromBytes(request_uri)
     print "New connection to:", parsedUri.host, parsedUri.port
     connect(parsedUri.host, parsedUri.port,
             self.clientFactory(self, parsedUri.toBytes()))
     HTTPParser.allHeadersReceived(self) # self.switchToBodyMode(None)
Example #6
0
    def request(self, method, uri, headers=None, bodyProducer=None, follow_redirects=2):
        """
        Issue a request to the server indicated by the given C{uri}.

        An existing connection from the connection pool may be used or a new one may be created.

        I{HTTP} and I{HTTPS} schemes are supported in C{uri}.

        @see: L{twisted.web.iweb.IAgent.request}
        """
        if uri.startswith('unix://'):
            unix_uri = re.match('^unix://(.*)//(.*)', uri)

            if not unix_uri:
                raise ValueError('Unix pipe http uri format is incorrect.')

            filename = '/%s' % unix_uri.group(1)
            endpoint = UNIXClientEndpoint(self._reactor, filename)

            parsedURI = _URI.fromBytes('unix://unix/%s' % unix_uri.group(2))
            parsedURI.host = unix_uri.group(1)

            key = (parsedURI.scheme, parsedURI.host, parsedURI.port)

            return self._requestWithEndpoint(key, endpoint, method, parsedURI,
                                         headers, bodyProducer,
                                         parsedURI.originForm)

        return super(UNIXAwareHttpAgent, self).request(method, uri, headers, bodyProducer)
Example #7
0
 def massage_url(url):
     if url is None or not isinstance(url, basestring):
         return None
     p = _URI.fromBytes(url)
     n = p.netloc.split('.')
     n.reverse()
     p.scheme = p.netloc = None
     return (','.join(n) + ')' + p.toBytes()).lower()
Example #8
0
def download_file(url, filename, callback=None, headers=None,
                  force_filename=False, allow_compression=True):
    """
    Downloads a file from a specific URL and returns a Deferred.  You can also
    specify a callback function to be called as parts are received.

    :param url: the url to download from
    :type url: string
    :param filename: the filename to save the file as
    :type filename: string
    :param callback: a function to be called when a part of data is received,
         it's signature should be: func(data, current_length, total_length)
    :type callback: function
    :param headers: any optional headers to send
    :type headers: dictionary
    :param force_filename: force us to use the filename specified rather than
                           one the server may suggest
    :type force_filename: boolean
    :param allow_compression: allows gzip & deflate decoding
    :type allow_compression: boolean

    :returns: the filename of the downloaded file
    :rtype: Deferred

    :raises t.w.e.PageRedirect: when server responds with a temporary redirect
         or permanently moved.
    :raises t.w.e.Error: for all other HTTP response errors (besides OK)
    """
    url = str(url)
    filename = str(filename)
    if headers:
        for key, value in headers.items():
            headers[str(key)] = str(value)

    if allow_compression:
        if not headers:
            headers = {}
        headers["accept-encoding"] = "deflate, gzip, x-gzip"

    # In twisted 13.1.0 the _parse() function was replaced by the _URI class 
    if hasattr(client, '_parse'):
        scheme, host, port, path = client._parse(url)
    else:
        from twisted.web.client import _URI
        uri = _URI.fromBytes(url)
        scheme = uri.scheme
        host = uri.host
        port = uri.port
        path = uri.path

    factory = HTTPDownloader(url, filename, callback, headers, force_filename, allow_compression)
    if scheme == "https":
        from twisted.internet import ssl
        reactor.connectSSL(host, port, factory, ssl.ClientContextFactory())
    else:
        reactor.connectTCP(host, port, factory)

    return factory.deferred
Example #9
0
	def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
		if hasattr(client, '_parse'):
			scheme, host, port, path = client._parse(url)
		else:
			from twisted.web.client import _URI
			uri = _URI.fromBytes(url)
			scheme = uri.scheme
			host = uri.host
			port = uri.port
			path = uri.path
		self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
		self.connection = reactor.connectTCP(host, port, self.factory)
Example #10
0
	def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
		if hasattr(client, '_parse'):
			scheme, host, port, path = client._parse(url)
		else:
			from twisted.web.client import _URI
			uri = _URI.fromBytes(url)
			scheme = uri.scheme
			host = uri.host
			port = uri.port
			path = uri.path

		self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
		self.connection = reactor.connectTCP(host, port, self.factory)
def sendUrlCommand(url, contextFactory=None, timeout=60, *args, **kwargs):
     if hasattr(client, '_parse'):
	scheme, host, port, path = client._parse(url)
     else:
	from twisted.web.client import _URI
	uri = _URI.fromBytes(url)
	scheme = uri.scheme
	host = uri.host
	port = uri.port
	path = uri.path
	factory = myHTTPClientFactory(url, *args, **kwargs)
	# print "scheme=%s host=%s port=%s path=%s\n" % (scheme, host, port, path)
	reactor.connectTCP(host, port, factory, timeout=timeout)
	return factory.deferred
Example #12
0
    def get_raw_page(url, *args, **kwargs):
        # In Twisted 13.1.0 _parse() function replaced by _URI class.
        # In Twisted 15.0.0 _URI class renamed to URI.
        if hasattr(client, "_parse"):
            scheme, host, port, path = client._parse(url)
        else:
            try:
                from twisted.web.client import _URI as URI
            except ImportError:
                from twisted.web.client import URI

            uri = URI.fromBytes(url)
            scheme = uri.scheme
            host = uri.host
            port = uri.port

        factory = client.HTTPClientFactory(url, *args, **kwargs)
        reactor.connectTCP(host, port, factory)
        return factory
Example #13
0
    def get_raw_page(url, *args, **kwargs):
        # In Twisted 13.1.0 _parse() function replaced by _URI class.
        # In Twisted 15.0.0 _URI class renamed to URI.
        if hasattr(client, "_parse"):
            scheme, host, port, path = client._parse(url)
        else:
            try:
                from twisted.web.client import _URI as URI
            except ImportError:
                from twisted.web.client import URI

            uri = URI.fromBytes(url)
            scheme = uri.scheme
            host = uri.host
            port = uri.port

        factory = client.HTTPClientFactory(url, *args, **kwargs)
        reactor.connectTCP(host, port, factory)
        return factory
Example #14
0
def getPage(url, contextFactory=None, *args, **kwargs):
	if hasattr(client, '_parse'):
		scheme, host, port, path = _parse(url)
	else:
			from twisted.web.client import _URI
			uri = _URI.fromBytes(url)
			scheme = uri.scheme
			host = uri.host
			port = uri.port
			path = uri.path
	factory = LimitedHTTPClientFactory(url, *args, **kwargs)
	if scheme == 'https':
		from twisted.internet import ssl 
		if contextFactory is None:
			contextFactory = ssl.ClientContextFactory()
		reactor.connectSSL(host, port, factory, contextFactory)
	else:
		reactor.connectTCP(host, port, factory)
	return factory.deferred
Example #15
0
 def __init__(self, url, outputfile, contextFactory = None, *args, **kwargs):
     if hasattr(client, '_parse'):
         scheme, host, port, path = client._parse(url)
     else:
         from twisted.web.client import _URI
         uri = _URI.fromBytes(url)
         scheme = uri.scheme
         host = uri.host
         port = uri.port
         path = uri.path
     self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
     if scheme == 'https':
         from twisted.internet import ssl
         if contextFactory is None:
             contextFactory = ssl.ClientContextFactory()
         self.connection = reactor.connectSSL(host, port, self.factory, contextFactory)
     else:
         self.connection = reactor.connectTCP(host, port, self.factory)
     return
Example #16
0
	def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
		if hasattr(client, '_parse'):
			scheme, host, port, path = client._parse(url)
		else:
			# _URI class renamed to URI in 15.0.0
 			try:
 				from twisted.web.client import _URI as URI
 			except ImportError:
 				from twisted.web.client import URI
 			uri = URI.fromBytes(url)
			scheme = uri.scheme
			host = uri.host
			port = uri.port
			path = uri.path

		self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
		if scheme == "https":
			self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
		else:
			self.connection = reactor.connectTCP(host, port, self.factory)
Example #17
0
    def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
        if hasattr(client, "_parse"):
            scheme, host, port, path = client._parse(url)
        else:
            # _URI class renamed to URI in 15.0.0
            try:
                from twisted.web.client import _URI as URI
            except ImportError:
                from twisted.web.client import URI
            uri = URI.fromBytes(url)
            scheme = uri.scheme
            host = uri.host
            port = uri.port
            path = uri.path

        self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
        if scheme == "https":
            self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
        else:
            self.connection = reactor.connectTCP(host, port, self.factory)
def sendPartnerBoxWebCommand(url, contextFactory=None, timeout=60, username = "******", password = "", *args, **kwargs):
	if hasattr(client, '_parse'):
		scheme, host, port, path = _parse(url)
	else:
			from twisted.web.client import _URI
			uri = _URI.fromBytes(url)
			scheme = uri.scheme
			host = uri.host
			port = uri.port
			path = uri.path
	basicAuth = encodestring(("%s:%s")%(username,password))
	authHeader = "Basic " + basicAuth.strip()
	AuthHeaders = {"Authorization": authHeader}
	if kwargs.has_key("headers"):
		kwargs["headers"].update(AuthHeaders)
	else:
		kwargs["headers"] = AuthHeaders
	factory = myHTTPClientFactory(url, *args, **kwargs)
	reactor.connectTCP(host, port, factory, timeout=timeout)
	return factory.deferred
Example #19
0
	def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
#<<<<<<< HEAD
		if hasattr(client, '_parse'):
			scheme, host, port, path = client._parse(url)
		else:
			from twisted.web.client import _URI
			uri = _URI.fromBytes(url)
			scheme = uri.scheme
			host = uri.host
			port = uri.port
			path = uri.path
# ======= another twisted fix possibility
#		parsed = urlparse(url)
#		scheme = parsed.scheme
#		host = parsed.hostname
#		port = parsed.port or (443 if scheme == 'https' else 80)

		self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
		if scheme == "https":
			self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
		else:
			self.connection = reactor.connectTCP(host, port, self.factory)
Example #20
0
    def request(self, method, uri, headers=None, bodyProducer=None):
        """
        Issue a request to the server indicated by the given C{uri}.

        An existing connection from the connection pool may be used or a new one may be created.
        Without additional modifications this connection pool may not be very useful because
        each connection in the pool will use the same Tor circuit.

        Currently only the I{HTTP} scheme is supported in C{uri}.

        @see: L{twisted.web.iweb.IAgent.request}
        """
        parsedURI = _URI.fromBytes(uri)
        endpoint = self._getEndpoint(parsedURI.scheme, parsedURI.host,
                                         parsedURI.port)

        # XXX
        # perhaps the request method should take a key?
        key = (parsedURI.scheme, parsedURI.host, parsedURI.port)

        return self._requestWithEndpoint(key, endpoint, method, parsedURI,
                                         headers, bodyProducer, parsedURI.originForm)
Example #21
0
	def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
#<<<<<<< HEAD
		if hasattr(client, '_parse'):
			scheme, host, port, path = client._parse(url)
		else:
			from twisted.web.client import _URI
			uri = _URI.fromBytes(url)
			scheme = uri.scheme
			host = uri.host
			port = uri.port
			path = uri.path
# ======= another twisted fix possibility
#		parsed = urlparse(url)
#		scheme = parsed.scheme
#		host = parsed.hostname
#		port = parsed.port or (443 if scheme == 'https' else 80)

		self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
		if scheme == "https":
			self.connection = reactor.connectSSL(host, port, self.factory, ssl.ClientContextFactory())
		else:
			self.connection = reactor.connectTCP(host, port, self.factory)
Example #22
0
def getPageCached(url, contextFactory=None, *args, **kwargs):
    """download a web page as a string, keep a cache of already downloaded pages

    Download a page. Return a deferred, which will callback with a
    page (as a string) or errback with a description of the error.

    See HTTPClientCacheFactory to see what extra args can be passed.
    """
    uri = _URI.fromBytes(url)
    scheme = uri.scheme
    host = uri.host
    port = uri.port

    factory = HTTPClientCacheFactory(url, *args, **kwargs)

    if scheme == 'https':
        if contextFactory is None:
            contextFactory = HTTPSVerifyingContextFactory(host)
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)

    return factory.deferred
Example #23
0
def getPageCached(url, contextFactory=None, *args, **kwargs):
    """download a web page as a string, keep a cache of already downloaded pages

    Download a page. Return a deferred, which will callback with a
    page (as a string) or errback with a description of the error.

    See HTTPClientCacheFactory to see what extra args can be passed.
    """
    uri = _URI.fromBytes(url)
    scheme = uri.scheme
    host = uri.host
    port = uri.port

    factory = HTTPClientCacheFactory(url, *args, **kwargs)

    if scheme == 'https':
        if contextFactory is None:
            contextFactory = HTTPSVerifyingContextFactory(host)
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)

    return factory.deferred
Example #24
0
    def __init__(self, url, outputfile, contextFactory=None, *args, **kwargs):
        if hasattr(client, '_parse'):
            scheme, host, port, path = client._parse(url)
        else:
            try:
                from twisted.web.client import _URI as URI
            except ImportError:
                from twisted.web.client import URI
            uri = URI.fromBytes(url)
            scheme = uri.scheme
            host = uri.host
            port = uri.port or (443 if scheme == 'https' else 80)
            path = uri.path

        self.factory = HTTPProgressDownloader(url, outputfile, *args, **kwargs)
        if scheme == "https":
            from twisted.internet import ssl
            if contextFactory is None:
                contextFactory = ssl.ClientContextFactory()
            self.connection = reactor.connectSSL(host, port, self.factory,
                                                 contextFactory)
        else:
            self.connection = reactor.connectTCP(host, port, self.factory)
Example #25
0
def getPageWithHeader(url, contextFactory=None, *args, **kwargs):
    """
    Same as twisted.web.client.getPage, but we keep the HTTP header in the
    result thanks to the HTTPClientFactoryWithHeader class
    """
    if parseAvailable:
        scheme, host, port, path = _parse(url)
    else:
        uri = _URI.fromBytes(url)
        scheme = uri.scheme
        host = uri.host
        port = uri.port

    factory = HTTPClientFactoryWithHeader(url, *args, **kwargs)
    d = factory.deferred

    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)
    return d
Example #26
0
def getPageWithHeader(url, contextFactory=None, *args, **kwargs):
    """
    Same as twisted.web.client.getPage, but we keep the HTTP header in the
    result thanks to the HTTPClientFactoryWithHeader class
    """
    if parseAvailable:
        scheme, host, port, path = _parse(url)
    else:
        uri = URI.fromBytes(url)
        scheme = uri.scheme
        host = uri.host
        port = uri.port

    factory = HTTPClientFactoryWithHeader(url, *args, **kwargs)
    d = factory.deferred

    if scheme == 'https':
        from twisted.internet import ssl
        if contextFactory is None:
            contextFactory = ssl.ClientContextFactory()
        reactor.connectSSL(host, port, factory, contextFactory)
    else:
        reactor.connectTCP(host, port, factory)
    return d
 def convertUriToRelative(uri):
     """ Converts an absolute URI to a relative one """
     parsedURI = _URI.fromBytes(uri)
     parsedURI.scheme = parsedURI.netloc = None
     return parsedURI.toBytes()
Example #28
0
 def uriEquals(self, uri, ignoreScheme=False):
     self_uri = _URI.fromBytes(self.uri)
     comp_uri = _URI.fromBytes(uri)
     if ignoreScheme:
         self_uri.scheme = comp_uri.scheme = None
     return self_uri.toBytes() == comp_uri.toBytes()
Example #29
0
def _download_file(url,
                   filename,
                   callback=None,
                   headers=None,
                   force_filename=False,
                   allow_compression=True):
    """
    Downloads a file from a specific URL and returns a Deferred. A callback
    function can be specified to be called as parts are received.

    Args:
        url (str): The url to download from
        filename (str): The filename to save the file as
        callback (func): A function to be called when a part of data is received,
            it's signature should be: func(data, current_length, total_length)
        headers (dict): Any optional headers to send
        force_filename (bool): force us to use the filename specified rather than
            one the server may suggest
        allow_compression (bool): Allows gzip & deflate decoding

    Returns:
        Deferred: the filename of the downloaded file

    Raises:
        t.w.e.PageRedirect
        t.w.e.Error: for all other HTTP response errors

    """

    if allow_compression:
        if not headers:
            headers = {}
        headers['accept-encoding'] = 'deflate, gzip, x-gzip'

    url = url.encode('utf8')
    filename = filename.encode('utf8')
    headers = utf8_encode_structure(headers) if headers else headers
    factory = HTTPDownloader(url, filename, callback, headers, force_filename,
                             allow_compression)

    # In Twisted 13.1.0 _parse() function replaced by _URI class.
    # In Twisted 15.0.0 _URI class renamed to URI.
    if hasattr(client, '_parse'):
        scheme, host, port, dummy_path = client._parse(url)
    else:
        try:
            from twisted.web.client import _URI as URI
        except ImportError:
            from twisted.web.client import URI
        finally:
            uri = URI.fromBytes(url)
            scheme = uri.scheme
            host = uri.host
            port = uri.port

    if scheme == 'https':
        from twisted.internet import ssl
        # ClientTLSOptions in Twisted >= 14, see ticket #2765 for details on this addition.
        try:
            from twisted.internet._sslverify import ClientTLSOptions
        except ImportError:
            ctx_factory = ssl.ClientContextFactory()
        else:

            class TLSSNIContextFactory(ssl.ClientContextFactory):  # pylint: disable=no-init
                """
                A custom context factory to add a server name for TLS connections.
                """
                def getContext(self):  # NOQA: N802
                    ctx = ssl.ClientContextFactory.getContext(self)
                    ClientTLSOptions(host, ctx)
                    return ctx

            ctx_factory = TLSSNIContextFactory()

        reactor.connectSSL(host, port, factory, ctx_factory)
    else:
        reactor.connectTCP(host, port, factory)

    return factory.deferred
Example #30
0
def _download_file(url, filename, callback=None, headers=None, force_filename=False, allow_compression=True):
    """
    Downloads a file from a specific URL and returns a Deferred. A callback
    function can be specified to be called as parts are received.

    Args:
        url (str): The url to download from
        filename (str): The filename to save the file as
        callback (func): A function to be called when a part of data is received,
            it's signature should be: func(data, current_length, total_length)
        headers (dict): Any optional headers to send
        force_filename (bool): force us to use the filename specified rather than
            one the server may suggest
        allow_compression (bool): Allows gzip & deflate decoding

    Returns:
        Deferred: the filename of the downloaded file

    Raises:
        t.w.e.PageRedirect
        t.w.e.Error: for all other HTTP response errors

    """

    if allow_compression:
        if not headers:
            headers = {}
        headers['accept-encoding'] = 'deflate, gzip, x-gzip'

    url = url.encode('utf8')
    filename = filename.encode('utf8')
    headers = utf8_encode_structure(headers) if headers else headers
    factory = HTTPDownloader(url, filename, callback, headers, force_filename, allow_compression)

    # In Twisted 13.1.0 _parse() function replaced by _URI class.
    # In Twisted 15.0.0 _URI class renamed to URI.
    if hasattr(client, '_parse'):
        scheme, host, port, dummy_path = client._parse(url)
    else:
        try:
            from twisted.web.client import _URI as URI
        except ImportError:
            from twisted.web.client import URI
        finally:
            uri = URI.fromBytes(url)
            scheme = uri.scheme
            host = uri.host
            port = uri.port

    if scheme == 'https':
        from twisted.internet import ssl
        # ClientTLSOptions in Twisted >= 14, see ticket #2765 for details on this addition.
        try:
            from twisted.internet._sslverify import ClientTLSOptions
        except ImportError:
            ctx_factory = ssl.ClientContextFactory()
        else:
            class TLSSNIContextFactory(ssl.ClientContextFactory):  # pylint: disable=no-init
                """
                A custom context factory to add a server name for TLS connections.
                """
                def getContext(self):  # NOQA: N802
                    ctx = ssl.ClientContextFactory.getContext(self)
                    ClientTLSOptions(host, ctx)
                    return ctx
            ctx_factory = TLSSNIContextFactory()

        reactor.connectSSL(host, port, factory, ctx_factory)
    else:
        reactor.connectTCP(host, port, factory)

    return factory.deferred
Example #31
0
def download_file(url, filename, callback=None, headers=None, force_filename=False, allow_compression=True):
    """
    Downloads a file from a specific URL and returns a Deferred.  You can also
    specify a callback function to be called as parts are received.

    :param url: the url to download from
    :type url: string
    :param filename: the filename to save the file as
    :type filename: string
    :param callback: a function to be called when a part of data is received,
         it's signature should be: func(data, current_length, total_length)
    :type callback: function
    :param headers: any optional headers to send
    :type headers: dictionary
    :param force_filename: force us to use the filename specified rather than
                           one the server may suggest
    :type force_filename: boolean
    :param allow_compression: allows gzip & deflate decoding
    :type allow_compression: boolean

    :returns: the filename of the downloaded file
    :rtype: Deferred

    :raises t.w.e.PageRedirect: when server responds with a temporary redirect
         or permanently moved.
    :raises t.w.e.Error: for all other HTTP response errors (besides OK)
    """
    url = str(url)
    filename = str(filename)
    if headers:
        for key, value in headers.items():
            headers[str(key)] = str(value)

    if allow_compression:
        if not headers:
            headers = {}
        headers["accept-encoding"] = "deflate, gzip, x-gzip"

    # In Twisted 13.1.0 _parse() function replaced by _URI class.
    # In Twisted 15.0.0 _URI class renamed to URI.
    if hasattr(client, "_parse"):
        scheme, host, port, path = client._parse(url)
    else:
        try:
            from twisted.web.client import _URI as URI
        except ImportError:
            from twisted.web.client import URI

        uri = URI.fromBytes(url)
        scheme = uri.scheme
        host = uri.host
        port = uri.port
        path = uri.path

    factory = HTTPDownloader(url, filename, callback, headers, force_filename, allow_compression)
    if scheme == "https":
        from twisted.internet import ssl

        # ClientTLSOptions in Twisted >= 14, see ticket #2765 for details on this addition.
        try:
            from twisted.internet._sslverify import ClientTLSOptions
        except ImportError:
            ctx_factory = ssl.ClientContextFactory()
        else:

            class TLSSNIContextFactory(ssl.ClientContextFactory):
                """
                A custom context factory to add a server name for TLS connections.
                """

                def getContext(self, hostname=None, port=None):
                    ctx = ssl.ClientContextFactory.getContext(self)
                    ClientTLSOptions(host, ctx)
                    return ctx

            ctx_factory = TLSSNIContextFactory()

        reactor.connectSSL(host, port, factory, ctx_factory)
    else:
        reactor.connectTCP(host, port, factory)

    return factory.deferred
 def convertUriToRelative(uri):
     """ Converts an absolute URI to a relative one """
     parsedURI = _URI.fromBytes(uri)
     parsedURI.scheme = parsedURI.netloc = None
     return parsedURI.toBytes()
Example #33
0
def download_file(url,
                  filename,
                  callback=None,
                  headers=None,
                  force_filename=False,
                  allow_compression=True):
    """
    Downloads a file from a specific URL and returns a Deferred.  You can also
    specify a callback function to be called as parts are received.

    :param url: the url to download from
    :type url: string
    :param filename: the filename to save the file as
    :type filename: string
    :param callback: a function to be called when a part of data is received,
         it's signature should be: func(data, current_length, total_length)
    :type callback: function
    :param headers: any optional headers to send
    :type headers: dictionary
    :param force_filename: force us to use the filename specified rather than
                           one the server may suggest
    :type force_filename: boolean
    :param allow_compression: allows gzip & deflate decoding
    :type allow_compression: boolean

    :returns: the filename of the downloaded file
    :rtype: Deferred

    :raises t.w.e.PageRedirect: when server responds with a temporary redirect
         or permanently moved.
    :raises t.w.e.Error: for all other HTTP response errors (besides OK)
    """
    url = str(url)
    filename = str(filename)
    if headers:
        for key, value in headers.items():
            headers[str(key)] = str(value)

    if allow_compression:
        if not headers:
            headers = {}
        headers["accept-encoding"] = "deflate, gzip, x-gzip"

    # In Twisted 13.1.0 _parse() function replaced by _URI class.
    # In Twisted 15.0.0 _URI class renamed to URI.
    if hasattr(client, "_parse"):
        scheme, host, port, path = client._parse(url)
    else:
        try:
            from twisted.web.client import _URI as URI
        except ImportError:
            from twisted.web.client import URI

        uri = URI.fromBytes(url)
        scheme = uri.scheme
        host = uri.host
        port = uri.port
        path = uri.path

    factory = HTTPDownloader(url, filename, callback, headers, force_filename,
                             allow_compression)
    if scheme == "https":
        from twisted.internet import ssl
        reactor.connectSSL(host, port, factory, ssl.ClientContextFactory())
    else:
        reactor.connectTCP(host, port, factory)

    return factory.deferred
Example #34
0
 def uriEquals(self, uri, ignoreScheme=False):
     self_uri = _URI.fromBytes(self.uri)
     comp_uri = _URI.fromBytes(uri)
     if ignoreScheme:
         self_uri.scheme = comp_uri.scheme = None
     return self_uri.toBytes() == comp_uri.toBytes()