def setUp(self): self.tmpname = self.mktemp() os.mkdir(self.tmpname) FilePath(self.tmpname).child("file").setContent(b"0123456789") r = static.File(self.tmpname) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) r.putChild(b"nolength", NoLengthResource()) r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) r.putChild(b"chunked", ChunkedResource()) r.putChild(b"broken-chunked", BrokenChunkedResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) r.putChild(b"largechunkedfile", LargeChunkedFileResource()) r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.host = 'localhost' if self.scheme == 'https': self.port = reactor.listenSSL(0, self.wrapper, ssl_context_factory( self.keyfile, self.certfile), interface=self.host) else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) self.download_request = self.download_handler.download_request
class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): # we try to use a cipher that is not enabled by default in OpenSSL custom_ciphers = 'CAMELLIA256-SHA' context_factory = ssl_context_factory(cipher_string=custom_ciphers) def testPayload(self): s = "0123456789" * 10 settings = Settings( {'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) return getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory).addCallback( self.assertEqual, to_bytes(s)) def testPayloadDisabledCipher(self): s = "0123456789" * 10 settings = Settings( {'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory) return self.assertFailure(d, OpenSSL.SSL.Error)
def setUp(self): self.tmpname = self.mktemp() os.mkdir(self.tmpname) FilePath(self.tmpname).child("file").setContent(b"0123456789") r = static.File(self.tmpname) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) r.putChild(b"nolength", NoLengthResource()) r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) r.putChild(b"chunked", ChunkedResource()) r.putChild(b"broken-chunked", BrokenChunkedResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) r.putChild(b"largechunkedfile", LargeChunkedFileResource()) r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.host = 'localhost' if self.scheme == 'https': self.port = reactor.listenSSL( 0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile), interface=self.host) else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port self.download_handler = self.download_handler_cls(Settings()) self.download_request = self.download_handler.download_request
def setUp(self): name = self.mktemp() os.mkdir(name) FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) r.putChild(b"nolength", NoLengthResource()) r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.host = 'localhost' if self.scheme == 'https': self.port = reactor.listenSSL(0, self.wrapper, ssl_context_factory(), interface=self.host) else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port self.download_handler = self.download_handler_cls(Settings()) self.download_request = self.download_handler.download_request
class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): # we try to use a cipher that is not enabled by default in OpenSSL custom_ciphers = 'CAMELLIA256-SHA' context_factory = ssl_context_factory(cipher_string=custom_ciphers) def testPayload(self): s = "0123456789" * 10 settings = Settings( {'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) return getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory).addCallback( self.assertEqual, to_bytes(s)) def testPayloadDisabledCipher(self): if sys.implementation.name == "pypy" and parse_version( cryptography.__version__) <= parse_version("2.3.1"): self.skipTest( "This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1" ) s = "0123456789" * 10 settings = Settings( {'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory) return self.assertFailure(d, OpenSSL.SSL.Error)
def setUp(self): # Initialize resource tree root = self._init_resource() self.site = Site(root, timeout=None) # Start server for testing self.hostname = 'localhost' context_factory = ssl_context_factory(self.key_file, self.certificate_file) server_endpoint = SSL4ServerEndpoint(reactor, 0, context_factory, interface=self.hostname) self.server = yield server_endpoint.listen(self.site) self.port_number = self.server.getHost().port # Connect H2 client with server self.client_certificate = get_client_certificate(self.key_file, self.certificate_file) client_options = optionsForClientTLS( hostname=self.hostname, trustRoot=self.client_certificate, acceptableProtocols=[b'h2'] ) uri = URI.fromBytes(bytes(self.get_url('/'), 'utf-8')) self.conn_closed_deferred = Deferred() from scrapy.core.http2.protocol import H2ClientFactory h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred) client_endpoint = SSL4ClientEndpoint(reactor, self.hostname, self.port_number, client_options) self.client = yield client_endpoint.connect(h2_client_factory)
def setUp(self): site = server.Site(UriResource(), timeout=None) self.port = reactor.listenSSL( 0, site, ssl_context_factory(self.keyfile, self.certfile), interface=self.host ) self.portno = self.port.getHost().port self.download_handler = create_instance(self.download_handler_cls, None, get_crawler()) self.download_request = self.download_handler.download_request
def setUp(self): self.tmpname = self.mktemp() os.mkdir(self.tmpname) FilePath(self.tmpname).child("file").setContent(b"0123456789") r = static.File(self.tmpname) self.site = server.Site(r, timeout=None) self.host = 'localhost' self.port = reactor.listenSSL( 0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), interface=self.host) self.portno = self.port.getHost().port crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}) self.download_handler = create_instance(self.download_handler_cls, None, crawler) self.download_request = self.download_handler.download_request
def setUp(self): name = self.mktemp() os.mkdir(name) FilePath(name).child("file").setContent(b"0123456789") r = static.File(name) r.putChild(b"redirect", util.Redirect(b"/file")) r.putChild(b"wait", ForeverTakingResource()) r.putChild(b"hang-after-headers", ForeverTakingResource(write=True)) r.putChild(b"nolength", NoLengthResource()) r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.host = "localhost" if self.scheme == "https": self.port = reactor.listenSSL(0, self.wrapper, ssl_context_factory(), interface=self.host) else: self.port = reactor.listenTCP(0, self.wrapper, interface=self.host) self.portno = self.port.getHost().port self.download_handler = self.download_handler_cls(Settings()) self.download_request = self.download_handler.download_request
def _listen(self, site): return reactor.listenSSL(0, site, contextFactory=self.context_factory or ssl_context_factory(), interface="127.0.0.1")