def _select_cipher_list(cipher_list=None): """ Selects either a passed in cipher list or uses the default openssl cipher list """ if cipher_list: return AcceptableCiphers.fromOpenSSLCipherString(cipher_list) if fips_manager.status(): return AcceptableCiphers.fromOpenSSLCipherString('TLSv1.2:kRSA:!eNULL:!aNULL') return AcceptableCiphers.fromOpenSSLCipherString(':'.join(DEFAULT_CIPHER_LIST))
def is_cipher_list(cipher_list): try: return len( AcceptableCiphers.fromOpenSSLCipherString( cipher_list)._ciphers) > 0 except Exception: return False
def get_context_factory(cert_path, pkey_path): """OpenSSL context factory. Generates an OpenSSL context factory using Twisted's CertificateOptions class. This will keep a server cipher order. Args: cert_path (string): The path to the certificate file pkey_path (string): The path to the private key file Returns: twisted.internet.ssl.CertificateOptions: An OpenSSL context factory """ with open(cert_path) as cert: certificate = Certificate.loadPEM(cert.read()).original with open(pkey_path) as pkey: private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original ciphers = AcceptableCiphers.fromOpenSSLCipherString(TLS_CIPHERS) cert_options = CertificateOptions( privateKey=private_key, certificate=certificate, raiseMinimumTo=TLSVersion.TLSv1_2, acceptableCiphers=ciphers, ) ctx = cert_options.getContext() ctx.use_certificate_chain_file(cert_path) ctx.set_options(SSL_OP_NO_RENEGOTIATION) return cert_options
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) self._ssl_method = method self.tls_verbose_logging = tls_verbose_logging if tls_ciphers: self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers) else: self.tls_ciphers = DEFAULT_CIPHERS
if self.verbose_logging: logger.debug('SSL connection to %s using protocol %s, cipher %s', self._hostnameASCII, connection.get_protocol_version_name(), connection.get_cipher_name(), ) server_cert = connection.get_peer_certificate() logger.debug('SSL connection certificate: issuer "%s", subject "%s"', x509name_to_string(server_cert.get_issuer()), x509name_to_string(server_cert.get_subject()), ) key_info = get_temp_key_info(connection._ssl) if key_info: logger.debug('SSL temp key: %s', key_info) try: verifyHostname(connection, self._hostnameASCII) except (CertificateError, VerificationError) as e: logger.warning( 'Remote certificate is not valid for hostname "%s"; %s', self._hostnameASCII, e) except ValueError as e: logger.warning( 'Ignoring error while verifying certificate ' 'from host "%s" (exception: %r)', self._hostnameASCII, e) DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
def _create_tls_server_context(config, cbdir, log): """ Create a CertificateOptions object for use with TLS listening endpoints. """ # server private key key_filepath = abspath(join(cbdir, config['key'])) log.info("Loading server TLS key from {key_filepath}", key_filepath=key_filepath) with open(key_filepath) as key_file: # server certificate (but only the server cert, no chain certs) cert_filepath = abspath(join(cbdir, config['certificate'])) log.info("Loading server TLS certificate from {cert_filepath}", cert_filepath=cert_filepath) with open(cert_filepath) as cert_file: key = KeyPair.load(key_file.read(), crypto.FILETYPE_PEM).original cert = Certificate.loadPEM(cert_file.read()).original # list of certificates that complete your verification chain extra_certs = None if 'chain_certificates' in config: extra_certs = [] for fname in config['chain_certificates']: extra_cert_filepath = abspath(join(cbdir, fname)) with open(extra_cert_filepath, 'r') as f: extra_certs.append(Certificate.loadPEM(f.read()).original) log.info("Loading server TLS chain certificate from {extra_cert_filepath}", extra_cert_filepath=extra_cert_filepath) # list of certificate authority certificate objects to use to verify the peer's certificate ca_certs = None if 'ca_certificates' in config: ca_certs = [] for fname in config['ca_certificates']: ca_cert_filepath = abspath(join(cbdir, fname)) with open(ca_cert_filepath, 'r') as f: ca_certs.append(Certificate.loadPEM(f.read()).original) log.info("Loading server TLS CA certificate from {ca_cert_filepath}", ca_cert_filepath=ca_cert_filepath) # ciphers we accept # # We prefer to make every single cipher (6 in total) _explicit_ (to reduce chances either we or the pattern-matching # language inside OpenSSL messes up) and drop support for Windows XP (we do WebSocket anyway). # # We don't use AES256 and SHA384, to reduce number of ciphers and since the additional # security gain seems not worth the additional performance drain. # # We also don't use ECDSA, since EC certificates a rare in the wild. # # The effective list of ciphers determined from an OpenSSL cipher string: # # openssl ciphers -v 'ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:' # # References: # # * https://www.ssllabs.com/ssltest/analyze.html?d=myserver.com # * http://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ # * http://www.openssl.org/docs/apps/ciphers.html#CIPHER_LIST_FORMAT # * https://wiki.mozilla.org/Talk:Security/Server_Side_TLS # if 'ciphers' in config: log.info("Using explicit TLS ciphers from config") crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString(config['ciphers']) else: log.info("Using secure default TLS ciphers") crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString( # AEAD modes (GCM) # 'ECDHE-ECDSA-AES128-GCM-SHA256:' 'ECDHE-RSA-AES128-GCM-SHA256:' # 'ECDHE-ECDSA-AES256-GCM-SHA384:' # 'ECDHE-RSA-AES256-GCM-SHA384:' 'DHE-RSA-AES128-GCM-SHA256:' # 'DHE-RSA-AES256-GCM-SHA384:' # CBC modes 'ECDHE-RSA-AES128-SHA256:' 'DHE-RSA-AES128-SHA256:' 'ECDHE-RSA-AES128-SHA:' 'DHE-RSA-AES128-SHA:' ) # DH modes require a parameter file if 'dhparam' in config: dhpath = FilePath(abspath(join(cbdir, config['dhparam']))) dh_params = DiffieHellmanParameters.fromFile(dhpath) else: dh_params = None log.warn("No OpenSSL DH parameter file set - DH cipher modes will be deactive!") # create a TLS context factory # see: https://twistedmatrix.com/documents/current/api/twisted.internet.ssl.CertificateOptions.html ctx = CertificateOptions( privateKey=key, certificate=cert, extraCertChain=extra_certs, verify=(ca_certs is not None), caCerts=ca_certs, dhParameters=dh_params, acceptableCiphers=crossbar_ciphers, # TLS hardening method=TLSv1_2_METHOD, enableSingleUseKeys=True, enableSessions=False, enableSessionTickets=False, fixBrokenPeers=False, ) # Without a curve being set, ECDH won't be available even if listed # in acceptable ciphers! # # The curves available in OpenSSL can be listed: # # openssl ecparam -list_curves # # prime256v1: X9.62/SECG curve over a 256 bit prime field # # This is elliptic curve "NIST P-256" from here # http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-4.pdf # # This seems to be the most widely used curve # # http://crypto.stackexchange.com/questions/11310/with-openssl-and-ecdhe-how-to-show-the-actual-curve-being-used # # and researchers think it is "ok" (other than wrt timing attacks etc) # # https://twitter.com/hyperelliptic/status/394258454342148096 # if ctx._ecCurve is None: log.warn("No OpenSSL elliptic curve set - EC cipher modes will be deactive!") else: if ctx._ecCurve.snName != "prime256v1": log.info("OpenSSL is using elliptic curve {curve}", curve=ctx._ecCurve.snName) else: log.info("OpenSSL is using elliptic curve prime256v1 (NIST P-256)") return ctx
def _create_tls_server_context(config, cbdir, log): """ Create a CertificateOptions object for use with TLS listening endpoints. """ # server private key key_filepath = abspath(join(cbdir, config['key'])) log.info("Loading server TLS key from {key_filepath}", key_filepath=key_filepath) with open(key_filepath) as key_file: # server certificate (but only the server cert, no chain certs) cert_filepath = abspath(join(cbdir, config['certificate'])) log.info("Loading server TLS certificate from {cert_filepath}", cert_filepath=cert_filepath) with open(cert_filepath) as cert_file: key = KeyPair.load(key_file.read(), crypto.FILETYPE_PEM).original cert = Certificate.loadPEM(cert_file.read()).original # list of certificates that complete your verification chain extra_certs = None if 'chain_certificates' in config: extra_certs = [] for fname in config['chain_certificates']: extra_cert_filepath = abspath(join(cbdir, fname)) with open(extra_cert_filepath, 'r') as f: extra_certs.append(Certificate.loadPEM(f.read()).original) log.info( "Loading server TLS chain certificate from {extra_cert_filepath}", extra_cert_filepath=extra_cert_filepath) # list of certificate authority certificate objects to use to verify the peer's certificate ca_certs = None if 'ca_certificates' in config: ca_certs = [] for fname in config['ca_certificates']: ca_cert_filepath = abspath(join(cbdir, fname)) with open(ca_cert_filepath, 'r') as f: ca_certs.append(Certificate.loadPEM(f.read()).original) log.info( "Loading server TLS CA certificate from {ca_cert_filepath}", ca_cert_filepath=ca_cert_filepath) # ciphers we accept # # We prefer to make every single cipher (6 in total) _explicit_ (to reduce chances either we or the pattern-matching # language inside OpenSSL messes up) and drop support for Windows XP (we do WebSocket anyway). # # We don't use AES256 and SHA384, to reduce number of ciphers and since the additional # security gain seems not worth the additional performance drain. # # We also don't use ECDSA, since EC certificates a rare in the wild. # # The effective list of ciphers determined from an OpenSSL cipher string: # # openssl ciphers -v 'ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:' # # References: # # * https://www.ssllabs.com/ssltest/analyze.html?d=myserver.com # * http://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ # * http://www.openssl.org/docs/apps/ciphers.html#CIPHER_LIST_FORMAT # * https://wiki.mozilla.org/Talk:Security/Server_Side_TLS # if 'ciphers' in config: log.info("Using explicit TLS ciphers from config") crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString( config['ciphers']) else: log.info("Using secure default TLS ciphers") crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString( # AEAD modes (GCM) # 'ECDHE-ECDSA-AES128-GCM-SHA256:' 'ECDHE-RSA-AES128-GCM-SHA256:' # 'ECDHE-ECDSA-AES256-GCM-SHA384:' # 'ECDHE-RSA-AES256-GCM-SHA384:' 'DHE-RSA-AES128-GCM-SHA256:' # 'DHE-RSA-AES256-GCM-SHA384:' # CBC modes 'ECDHE-RSA-AES128-SHA256:' 'DHE-RSA-AES128-SHA256:' 'ECDHE-RSA-AES128-SHA:' 'DHE-RSA-AES128-SHA:') # DH modes require a parameter file if 'dhparam' in config: dhpath = FilePath(abspath(join(cbdir, config['dhparam']))) dh_params = DiffieHellmanParameters.fromFile(dhpath) else: dh_params = None log.warn( "No OpenSSL DH parameter file set - DH cipher modes will be deactive!" ) ctx = CertificateOptions( privateKey=key, certificate=cert, extraCertChain=extra_certs, verify=(ca_certs is not None), caCerts=ca_certs, dhParameters=dh_params, acceptableCiphers=crossbar_ciphers, # Disable SSLv3 and TLSv1 -- only allow TLSv1.1 or higher # # We are using Twisted private stuff (from twisted.internet._sslverify import TLSVersion), # as OpenSSL.SSL.TLSv1_1_METHOD wont work: # # [ERROR] File "../twisted/internet/_sslverify.py", line 1530, in __init__ # if raiseMinimumTo > self._defaultMinimumTLSVersion: # builtins.TypeError: '>' not supported between instances of 'int' and 'NamedConstant' # raiseMinimumTo=TLSVersion.TLSv1_1, # TLS hardening enableSingleUseKeys=True, enableSessions=False, enableSessionTickets=False, fixBrokenPeers=False, ) # Without a curve being set, ECDH won't be available even if listed # in acceptable ciphers! # # The curves available in OpenSSL can be listed: # # openssl ecparam -list_curves # # prime256v1: X9.62/SECG curve over a 256 bit prime field # # This is elliptic curve "NIST P-256" from here # http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-4.pdf # # This seems to be the most widely used curve # # http://crypto.stackexchange.com/questions/11310/with-openssl-and-ecdhe-how-to-show-the-actual-curve-being-used # # and researchers think it is "ok" (other than wrt timing attacks etc) # # https://twitter.com/hyperelliptic/status/394258454342148096 # if False: # FIXME: this doesn't work anymore with Twisted 18.4. (there now seems more complex machinery # in self._ecChooser) if ctx._ecCurve is None: log.warn( "No OpenSSL elliptic curve set - EC cipher modes will be deactive!" ) else: if ctx._ecCurve.snName != "prime256v1": log.info("OpenSSL is using elliptic curve {curve}", curve=ctx._ecCurve.snName) else: log.info( "OpenSSL is using elliptic curve prime256v1 (NIST P-256)") return ctx
#coding:utf-8 # 解决https网站的反扒问题 from twisted.internet.ssl import AcceptableCiphers from scrapy.core.downloader import contextfactory import scrapy from ahui.items import AhuiItem contextfactory.DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString( 'DEFAULT:!DH') class ygo(scrapy.Spider): name = 'ygo' allowed_domains = ["www.ygo-sem.cn"] # 随机取一个开始的页面,然后通过下一页去获取其他的页面 start_urls = ['https://www.ygo-sem.cn/html/7/6/21251800.html'] def parse(self, response): item = AhuiItem() # 实例化item类 # 解析所需要的数据 card_img = response.css( '#card_frame #card_image_1::attr(src)').extract_first() card_info = response.css('.item_box') card_list = [ item.css('.item_box_value::text').extract_first().strip() for item in card_info ] # 按照位置,0是中文名,1是日文名,2是英文名,3是属性,4是种族,5是星级,6是怪兽类型,7是卡片code,8是攻击力,9是守备力 print(card_list) if len(card_list) == 10: item['card_name_zn'] = card_list[0] item['card_name_ja'] = card_list[1]
class ScrapyClientTLSOptions(ClientTLSOptions): """ SSL Client connection creator ignoring certificate verification errors (for genuinely invalid certificates or bugs in verification code). Same as Twisted's private _sslverify.ClientTLSOptions, except that VerificationError and ValueError exceptions are caught, so that the connection is not closed, only logging warnings. """ def _identityVerifyingInfoCallback(self, connection, where, ret): if where & SSL_CB_HANDSHAKE_START: set_tlsext_host_name(connection, self._hostnameBytes) elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) except VerificationError as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) except ValueError as e: logger.warning( 'Ignoring error while verifying certificate ' 'from host "{}" (exception: {})'.format( self._hostnameASCII, repr(e))) DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
def create_listening_endpoint_from_config(config, cbdir, reactor): """ Create a Twisted stream server endpoint from a Crossbar.io transport configuration. See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IStreamServerEndpoint.html :param config: The transport configuration. :type config: dict :param cbdir: Crossbar.io node directory (we need this for TLS key/certificates). :type cbdir: str :param reactor: The reactor to use for endpoint creation. :type reactor: obj :returns obj -- An instance implementing IStreamServerEndpoint """ log = make_logger() endpoint = None # a TCP endpoint # if config['type'] == 'tcp': # the TCP protocol version (v4 or v6) # version = int(config.get('version', 4)) # the listening port # if type(config['port']) is six.text_type: # read port from environment variable .. try: port = int(environ[config['port'][1:]]) except Exception as e: print( "Could not read listening port from env var: {}".format(e)) raise e else: port = config['port'] # the listening interface # interface = str(config.get('interface', '').strip()) # the TCP accept queue depth # backlog = int(config.get('backlog', 50)) if 'tls' in config: if _HAS_TLS: key_filepath = abspath(join(cbdir, config['tls']['key'])) cert_filepath = abspath( join(cbdir, config['tls']['certificate'])) with open(key_filepath) as key_file: with open(cert_filepath) as cert_file: if 'dhparam' in config['tls']: dhpath = FilePath( abspath(join(cbdir, config['tls']['dhparam']))) dh_params = DiffieHellmanParameters.fromFile( dhpath) else: # XXX won't be doing ANY EDH # curves... maybe make dhparam required? # or do "whatever tlxctx was doing" dh_params = None log.warn( "OpenSSL DH modes not active (no 'dhparam')") # create a TLS context factory # key = key_file.read() cert = cert_file.read() ca_certs = None if 'ca_certificates' in config['tls']: ca_certs = [] for fname in config['tls']['ca_certificates']: with open(fname, 'r') as f: ca_certs.append( Certificate.loadPEM(f.read()).original) crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString( 'ECDHE-RSA-AES128-GCM-SHA256:' 'DHE-RSA-AES128-GCM-SHA256:' 'ECDHE-RSA-AES128-SHA256:' 'DHE-RSA-AES128-SHA256:' 'ECDHE-RSA-AES128-SHA:' 'DHE-RSA-AES128-SHA') ctx = CertificateOptions( privateKey=KeyPair.load( key, crypto.FILETYPE_PEM).original, certificate=Certificate.loadPEM(cert).original, verify=(ca_certs is not None), caCerts=ca_certs, dhParameters=dh_params, acceptableCiphers=crossbar_ciphers, ) if ctx._ecCurve is None: log.warn( "OpenSSL failed to set ECDH default curve") else: log.info( "Ok, OpenSSL is using ECDH elliptic curve {curve}", curve=ctx._ecCurve.snName, ) # create a TLS server endpoint # if version == 4: endpoint = SSL4ServerEndpoint(reactor, port, ctx, backlog=backlog, interface=interface) elif version == 6: raise Exception("TLS on IPv6 not implemented") else: raise Exception( "invalid TCP protocol version {}".format(version)) else: raise Exception( "TLS transport requested, but TLS packages not available:\n{}" .format(_LACKS_TLS_MSG)) else: # create a non-TLS server endpoint # if version == 4: endpoint = TCP4ServerEndpoint(reactor, port, backlog=backlog, interface=interface) elif version == 6: endpoint = TCP6ServerEndpoint(reactor, port, backlog=backlog, interface=interface) else: raise Exception( "invalid TCP protocol version {}".format(version)) # a Unix Domain Socket endpoint # elif config['type'] == 'unix': # the accept queue depth # backlog = int(config.get('backlog', 50)) # the path # path = FilePath(join(cbdir, config['path'])) # if there is already something there, delete it. # if path.exists(): log.info(("{path} exists, attempting to remove before using as a " "UNIX socket"), path=path) path.remove() # create the endpoint # endpoint = UNIXServerEndpoint(reactor, path.path, backlog=backlog) else: raise Exception("invalid endpoint type '{}'".format(config['type'])) return endpoint