def _select_cipher_list(cipher_list=None):
"""
Selects either a passed in cipher list or uses the default openssl cipher list
"""
if cipher_list:
return AcceptableCiphers.fromOpenSSLCipherString(cipher_list)
if fips_manager.status():
return AcceptableCiphers.fromOpenSSLCipherString('TLSv1.2:kRSA:!eNULL:!aNULL')
return AcceptableCiphers.fromOpenSSLCipherString(':'.join(DEFAULT_CIPHER_LIST))
def is_cipher_list(cipher_list):
try:
return len(
AcceptableCiphers.fromOpenSSLCipherString(
cipher_list)._ciphers) > 0
except Exception:
return False
def get_context_factory(cert_path, pkey_path):
"""OpenSSL context factory.
Generates an OpenSSL context factory using Twisted's CertificateOptions class.
This will keep a server cipher order.
Args:
cert_path (string): The path to the certificate file
pkey_path (string): The path to the private key file
Returns:
twisted.internet.ssl.CertificateOptions: An OpenSSL context factory
"""
with open(cert_path) as cert:
certificate = Certificate.loadPEM(cert.read()).original
with open(pkey_path) as pkey:
private_key = KeyPair.load(pkey.read(), FILETYPE_PEM).original
ciphers = AcceptableCiphers.fromOpenSSLCipherString(TLS_CIPHERS)
cert_options = CertificateOptions(
privateKey=private_key,
certificate=certificate,
raiseMinimumTo=TLSVersion.TLSv1_2,
acceptableCiphers=ciphers,
)
ctx = cert_options.getContext()
ctx.use_certificate_chain_file(cert_path)
ctx.set_options(SSL_OP_NO_RENEGOTIATION)
return cert_options
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
if tls_ciphers:
self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
else:
self.tls_ciphers = DEFAULT_CIPHERS
if self.verbose_logging:
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
server_cert = connection.get_peer_certificate()
logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
x509name_to_string(server_cert.get_issuer()),
x509name_to_string(server_cert.get_subject()),
)
key_info = get_temp_key_info(connection._ssl)
if key_info:
logger.debug('SSL temp key: %s', key_info)
try:
verifyHostname(connection, self._hostnameASCII)
except (CertificateError, VerificationError) as e:
logger.warning(
'Remote certificate is not valid for hostname "%s"; %s',
self._hostnameASCII, e)
except ValueError as e:
logger.warning(
'Ignoring error while verifying certificate '
'from host "%s" (exception: %r)',
self._hostnameASCII, e)
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
def _create_tls_server_context(config, cbdir, log):
"""
Create a CertificateOptions object for use with TLS listening endpoints.
"""
# server private key
key_filepath = abspath(join(cbdir, config['key']))
log.info("Loading server TLS key from {key_filepath}", key_filepath=key_filepath)
with open(key_filepath) as key_file:
# server certificate (but only the server cert, no chain certs)
cert_filepath = abspath(join(cbdir, config['certificate']))
log.info("Loading server TLS certificate from {cert_filepath}", cert_filepath=cert_filepath)
with open(cert_filepath) as cert_file:
key = KeyPair.load(key_file.read(), crypto.FILETYPE_PEM).original
cert = Certificate.loadPEM(cert_file.read()).original
# list of certificates that complete your verification chain
extra_certs = None
if 'chain_certificates' in config:
extra_certs = []
for fname in config['chain_certificates']:
extra_cert_filepath = abspath(join(cbdir, fname))
with open(extra_cert_filepath, 'r') as f:
extra_certs.append(Certificate.loadPEM(f.read()).original)
log.info("Loading server TLS chain certificate from {extra_cert_filepath}", extra_cert_filepath=extra_cert_filepath)
# list of certificate authority certificate objects to use to verify the peer's certificate
ca_certs = None
if 'ca_certificates' in config:
ca_certs = []
for fname in config['ca_certificates']:
ca_cert_filepath = abspath(join(cbdir, fname))
with open(ca_cert_filepath, 'r') as f:
ca_certs.append(Certificate.loadPEM(f.read()).original)
log.info("Loading server TLS CA certificate from {ca_cert_filepath}", ca_cert_filepath=ca_cert_filepath)
# ciphers we accept
#
# We prefer to make every single cipher (6 in total) _explicit_ (to reduce chances either we or the pattern-matching
# language inside OpenSSL messes up) and drop support for Windows XP (we do WebSocket anyway).
#
# We don't use AES256 and SHA384, to reduce number of ciphers and since the additional
# security gain seems not worth the additional performance drain.
#
# We also don't use ECDSA, since EC certificates a rare in the wild.
#
# The effective list of ciphers determined from an OpenSSL cipher string:
#
# openssl ciphers -v 'ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:'
#
# References:
#
# * https://www.ssllabs.com/ssltest/analyze.html?d=myserver.com
# * http://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
# * http://www.openssl.org/docs/apps/ciphers.html#CIPHER_LIST_FORMAT
# * https://wiki.mozilla.org/Talk:Security/Server_Side_TLS
#
if 'ciphers' in config:
log.info("Using explicit TLS ciphers from config")
crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString(config['ciphers'])
else:
log.info("Using secure default TLS ciphers")
crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString(
# AEAD modes (GCM)
# 'ECDHE-ECDSA-AES128-GCM-SHA256:'
'ECDHE-RSA-AES128-GCM-SHA256:'
# 'ECDHE-ECDSA-AES256-GCM-SHA384:'
# 'ECDHE-RSA-AES256-GCM-SHA384:'
'DHE-RSA-AES128-GCM-SHA256:'
# 'DHE-RSA-AES256-GCM-SHA384:'
# CBC modes
'ECDHE-RSA-AES128-SHA256:'
'DHE-RSA-AES128-SHA256:'
'ECDHE-RSA-AES128-SHA:'
'DHE-RSA-AES128-SHA:'
)
# DH modes require a parameter file
if 'dhparam' in config:
dhpath = FilePath(abspath(join(cbdir, config['dhparam'])))
dh_params = DiffieHellmanParameters.fromFile(dhpath)
else:
dh_params = None
log.warn("No OpenSSL DH parameter file set - DH cipher modes will be deactive!")
# create a TLS context factory
# see: https://twistedmatrix.com/documents/current/api/twisted.internet.ssl.CertificateOptions.html
ctx = CertificateOptions(
privateKey=key,
certificate=cert,
extraCertChain=extra_certs,
verify=(ca_certs is not None),
caCerts=ca_certs,
dhParameters=dh_params,
acceptableCiphers=crossbar_ciphers,
# TLS hardening
method=TLSv1_2_METHOD,
enableSingleUseKeys=True,
enableSessions=False,
enableSessionTickets=False,
fixBrokenPeers=False,
)
# Without a curve being set, ECDH won't be available even if listed
# in acceptable ciphers!
#
# The curves available in OpenSSL can be listed:
#
# openssl ecparam -list_curves
#
# prime256v1: X9.62/SECG curve over a 256 bit prime field
#
# This is elliptic curve "NIST P-256" from here
# http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-4.pdf
#
# This seems to be the most widely used curve
#
# http://crypto.stackexchange.com/questions/11310/with-openssl-and-ecdhe-how-to-show-the-actual-curve-being-used
#
# and researchers think it is "ok" (other than wrt timing attacks etc)
#
# https://twitter.com/hyperelliptic/status/394258454342148096
#
if ctx._ecCurve is None:
log.warn("No OpenSSL elliptic curve set - EC cipher modes will be deactive!")
else:
if ctx._ecCurve.snName != "prime256v1":
log.info("OpenSSL is using elliptic curve {curve}", curve=ctx._ecCurve.snName)
else:
log.info("OpenSSL is using elliptic curve prime256v1 (NIST P-256)")
return ctx
def _create_tls_server_context(config, cbdir, log):
"""
Create a CertificateOptions object for use with TLS listening endpoints.
"""
# server private key
key_filepath = abspath(join(cbdir, config['key']))
log.info("Loading server TLS key from {key_filepath}",
key_filepath=key_filepath)
with open(key_filepath) as key_file:
# server certificate (but only the server cert, no chain certs)
cert_filepath = abspath(join(cbdir, config['certificate']))
log.info("Loading server TLS certificate from {cert_filepath}",
cert_filepath=cert_filepath)
with open(cert_filepath) as cert_file:
key = KeyPair.load(key_file.read(), crypto.FILETYPE_PEM).original
cert = Certificate.loadPEM(cert_file.read()).original
# list of certificates that complete your verification chain
extra_certs = None
if 'chain_certificates' in config:
extra_certs = []
for fname in config['chain_certificates']:
extra_cert_filepath = abspath(join(cbdir, fname))
with open(extra_cert_filepath, 'r') as f:
extra_certs.append(Certificate.loadPEM(f.read()).original)
log.info(
"Loading server TLS chain certificate from {extra_cert_filepath}",
extra_cert_filepath=extra_cert_filepath)
# list of certificate authority certificate objects to use to verify the peer's certificate
ca_certs = None
if 'ca_certificates' in config:
ca_certs = []
for fname in config['ca_certificates']:
ca_cert_filepath = abspath(join(cbdir, fname))
with open(ca_cert_filepath, 'r') as f:
ca_certs.append(Certificate.loadPEM(f.read()).original)
log.info(
"Loading server TLS CA certificate from {ca_cert_filepath}",
ca_cert_filepath=ca_cert_filepath)
# ciphers we accept
#
# We prefer to make every single cipher (6 in total) _explicit_ (to reduce chances either we or the pattern-matching
# language inside OpenSSL messes up) and drop support for Windows XP (we do WebSocket anyway).
#
# We don't use AES256 and SHA384, to reduce number of ciphers and since the additional
# security gain seems not worth the additional performance drain.
#
# We also don't use ECDSA, since EC certificates a rare in the wild.
#
# The effective list of ciphers determined from an OpenSSL cipher string:
#
# openssl ciphers -v 'ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:'
#
# References:
#
# * https://www.ssllabs.com/ssltest/analyze.html?d=myserver.com
# * http://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
# * http://www.openssl.org/docs/apps/ciphers.html#CIPHER_LIST_FORMAT
# * https://wiki.mozilla.org/Talk:Security/Server_Side_TLS
#
if 'ciphers' in config:
log.info("Using explicit TLS ciphers from config")
crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString(
config['ciphers'])
else:
log.info("Using secure default TLS ciphers")
crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString(
# AEAD modes (GCM)
# 'ECDHE-ECDSA-AES128-GCM-SHA256:'
'ECDHE-RSA-AES128-GCM-SHA256:'
# 'ECDHE-ECDSA-AES256-GCM-SHA384:'
# 'ECDHE-RSA-AES256-GCM-SHA384:'
'DHE-RSA-AES128-GCM-SHA256:'
# 'DHE-RSA-AES256-GCM-SHA384:'
# CBC modes
'ECDHE-RSA-AES128-SHA256:'
'DHE-RSA-AES128-SHA256:'
'ECDHE-RSA-AES128-SHA:'
'DHE-RSA-AES128-SHA:')
# DH modes require a parameter file
if 'dhparam' in config:
dhpath = FilePath(abspath(join(cbdir, config['dhparam'])))
dh_params = DiffieHellmanParameters.fromFile(dhpath)
else:
dh_params = None
log.warn(
"No OpenSSL DH parameter file set - DH cipher modes will be deactive!"
)
ctx = CertificateOptions(
privateKey=key,
certificate=cert,
extraCertChain=extra_certs,
verify=(ca_certs is not None),
caCerts=ca_certs,
dhParameters=dh_params,
acceptableCiphers=crossbar_ciphers,
# Disable SSLv3 and TLSv1 -- only allow TLSv1.1 or higher
#
# We are using Twisted private stuff (from twisted.internet._sslverify import TLSVersion),
# as OpenSSL.SSL.TLSv1_1_METHOD wont work:
#
# [ERROR] File "../twisted/internet/_sslverify.py", line 1530, in __init__
# if raiseMinimumTo > self._defaultMinimumTLSVersion:
# builtins.TypeError: '>' not supported between instances of 'int' and 'NamedConstant'
#
raiseMinimumTo=TLSVersion.TLSv1_1,
# TLS hardening
enableSingleUseKeys=True,
enableSessions=False,
enableSessionTickets=False,
fixBrokenPeers=False,
)
# Without a curve being set, ECDH won't be available even if listed
# in acceptable ciphers!
#
# The curves available in OpenSSL can be listed:
#
# openssl ecparam -list_curves
#
# prime256v1: X9.62/SECG curve over a 256 bit prime field
#
# This is elliptic curve "NIST P-256" from here
# http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-4.pdf
#
# This seems to be the most widely used curve
#
# http://crypto.stackexchange.com/questions/11310/with-openssl-and-ecdhe-how-to-show-the-actual-curve-being-used
#
# and researchers think it is "ok" (other than wrt timing attacks etc)
#
# https://twitter.com/hyperelliptic/status/394258454342148096
#
if False:
# FIXME: this doesn't work anymore with Twisted 18.4. (there now seems more complex machinery
# in self._ecChooser)
if ctx._ecCurve is None:
log.warn(
"No OpenSSL elliptic curve set - EC cipher modes will be deactive!"
)
else:
if ctx._ecCurve.snName != "prime256v1":
log.info("OpenSSL is using elliptic curve {curve}",
curve=ctx._ecCurve.snName)
else:
log.info(
"OpenSSL is using elliptic curve prime256v1 (NIST P-256)")
return ctx
#coding:utf-8
# 解决https网站的反扒问题
from twisted.internet.ssl import AcceptableCiphers
from scrapy.core.downloader import contextfactory
import scrapy
from ahui.items import AhuiItem
contextfactory.DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString(
'DEFAULT:!DH')
class ygo(scrapy.Spider):
name = 'ygo'
allowed_domains = ["www.ygo-sem.cn"]
# 随机取一个开始的页面，然后通过下一页去获取其他的页面
start_urls = ['https://www.ygo-sem.cn/html/7/6/21251800.html']
def parse(self, response):
item = AhuiItem() # 实例化item类
# 解析所需要的数据
card_img = response.css(
'#card_frame #card_image_1::attr(src)').extract_first()
card_info = response.css('.item_box')
card_list = [
item.css('.item_box_value::text').extract_first().strip()
for item in card_info
]
# 按照位置，0是中文名，1是日文名，2是英文名，3是属性，4是种族，5是星级，6是怪兽类型，7是卡片code，8是攻击力，9是守备力
print(card_list)
if len(card_list) == 10:
item['card_name_zn'] = card_list[0]
item['card_name_ja'] = card_list[1]
class ScrapyClientTLSOptions(ClientTLSOptions):
"""
SSL Client connection creator ignoring certificate verification errors
(for genuinely invalid certificates or bugs in verification code).
Same as Twisted's private _sslverify.ClientTLSOptions,
except that VerificationError and ValueError exceptions are caught,
so that the connection is not closed, only logging warnings.
"""
def _identityVerifyingInfoCallback(self, connection, where, ret):
if where & SSL_CB_HANDSHAKE_START:
set_tlsext_host_name(connection, self._hostnameBytes)
elif where & SSL_CB_HANDSHAKE_DONE:
try:
verifyHostname(connection, self._hostnameASCII)
except VerificationError as e:
logger.warning(
'Remote certificate is not valid for hostname "{}"; {}'.format(
self._hostnameASCII, e))
except ValueError as e:
logger.warning(
'Ignoring error while verifying certificate '
'from host "{}" (exception: {})'.format(
self._hostnameASCII, repr(e)))
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
def create_listening_endpoint_from_config(config, cbdir, reactor):
"""
Create a Twisted stream server endpoint from a Crossbar.io transport configuration.
See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IStreamServerEndpoint.html
:param config: The transport configuration.
:type config: dict
:param cbdir: Crossbar.io node directory (we need this for TLS key/certificates).
:type cbdir: str
:param reactor: The reactor to use for endpoint creation.
:type reactor: obj
:returns obj -- An instance implementing IStreamServerEndpoint
"""
log = make_logger()
endpoint = None
# a TCP endpoint
#
if config['type'] == 'tcp':
# the TCP protocol version (v4 or v6)
#
version = int(config.get('version', 4))
# the listening port
#
if type(config['port']) is six.text_type:
# read port from environment variable ..
try:
port = int(environ[config['port'][1:]])
except Exception as e:
print(
"Could not read listening port from env var: {}".format(e))
raise e
else:
port = config['port']
# the listening interface
#
interface = str(config.get('interface', '').strip())
# the TCP accept queue depth
#
backlog = int(config.get('backlog', 50))
if 'tls' in config:
if _HAS_TLS:
key_filepath = abspath(join(cbdir, config['tls']['key']))
cert_filepath = abspath(
join(cbdir, config['tls']['certificate']))
with open(key_filepath) as key_file:
with open(cert_filepath) as cert_file:
if 'dhparam' in config['tls']:
dhpath = FilePath(
abspath(join(cbdir, config['tls']['dhparam'])))
dh_params = DiffieHellmanParameters.fromFile(
dhpath)
else:
# XXX won't be doing ANY EDH
# curves... maybe make dhparam required?
# or do "whatever tlxctx was doing"
dh_params = None
log.warn(
"OpenSSL DH modes not active (no 'dhparam')")
# create a TLS context factory
#
key = key_file.read()
cert = cert_file.read()
ca_certs = None
if 'ca_certificates' in config['tls']:
ca_certs = []
for fname in config['tls']['ca_certificates']:
with open(fname, 'r') as f:
ca_certs.append(
Certificate.loadPEM(f.read()).original)
crossbar_ciphers = AcceptableCiphers.fromOpenSSLCipherString(
'ECDHE-RSA-AES128-GCM-SHA256:'
'DHE-RSA-AES128-GCM-SHA256:'
'ECDHE-RSA-AES128-SHA256:'
'DHE-RSA-AES128-SHA256:'
'ECDHE-RSA-AES128-SHA:'
'DHE-RSA-AES128-SHA')
ctx = CertificateOptions(
privateKey=KeyPair.load(
key, crypto.FILETYPE_PEM).original,
certificate=Certificate.loadPEM(cert).original,
verify=(ca_certs is not None),
caCerts=ca_certs,
dhParameters=dh_params,
acceptableCiphers=crossbar_ciphers,
)
if ctx._ecCurve is None:
log.warn(
"OpenSSL failed to set ECDH default curve")
else:
log.info(
"Ok, OpenSSL is using ECDH elliptic curve {curve}",
curve=ctx._ecCurve.snName,
)
# create a TLS server endpoint
#
if version == 4:
endpoint = SSL4ServerEndpoint(reactor,
port,
ctx,
backlog=backlog,
interface=interface)
elif version == 6:
raise Exception("TLS on IPv6 not implemented")
else:
raise Exception(
"invalid TCP protocol version {}".format(version))
else:
raise Exception(
"TLS transport requested, but TLS packages not available:\n{}"
.format(_LACKS_TLS_MSG))
else:
# create a non-TLS server endpoint
#
if version == 4:
endpoint = TCP4ServerEndpoint(reactor,
port,
backlog=backlog,
interface=interface)
elif version == 6:
endpoint = TCP6ServerEndpoint(reactor,
port,
backlog=backlog,
interface=interface)
else:
raise Exception(
"invalid TCP protocol version {}".format(version))
# a Unix Domain Socket endpoint
#
elif config['type'] == 'unix':
# the accept queue depth
#
backlog = int(config.get('backlog', 50))
# the path
#
path = FilePath(join(cbdir, config['path']))
# if there is already something there, delete it.
#
if path.exists():
log.info(("{path} exists, attempting to remove before using as a "
"UNIX socket"),
path=path)
path.remove()
# create the endpoint
#
endpoint = UNIXServerEndpoint(reactor, path.path, backlog=backlog)
else:
raise Exception("invalid endpoint type '{}'".format(config['type']))
return endpoint
