def start(self):
if not len(self.al.spiders):
logger.error("No Spiders loaded. exit.")
sys.exit(1)
else:
message = "Loaded spiders: "
for s in self.al.spiders:
message += str(s.__class__).split(".")[-1].split("'")[0] + ", "
logger.info(message.strip(", "))
# 创建线程池
if self.spider_threads:
self.tp = ThreadPool(self.spider_threads)
else:
self.tp = ThreadPool()
for sp in self.al.spiders:
# 将spider中的run方法添加到线程池中
self.tp.add_function(sp.run)
# 开始线程池
self.tp.run(join=False)
# 输出结果
self.sd = SaveData(self.al.results,
self.tp,
use_file=self.output_file,
use_database=self.output_db,
filename=self.output_filename)
if self.save_data_threads:
self.write_file_tp = ThreadPool(self.save_data_threads)
else:
self.write_file_tp = ThreadPool()
self.write_file_tp = ThreadPool()
self.write_file_tp.add_function(self.sd.write)
self.write_file_tp.run()
def _command_resum_rate(self, target):
"""
Performs 100 session resumptions with the server in order to estimate
the session resumption rate.
"""
# Create a thread pool and process the jobs
NB_THREADS = 20
MAX_RESUM = 100
thread_pool = ThreadPool()
for _ in xrange(MAX_RESUM):
thread_pool.add_job((self._resume_with_session_id, (target, )))
thread_pool.start(NB_THREADS)
# Format session ID results
(txt_resum,
xml_resum) = self._format_resum_id_results(thread_pool, MAX_RESUM)
# Text output
cmd_title = 'Resumption Rate with Session IDs'
txt_result = [self.PLUGIN_TITLE_FORMAT(cmd_title) + ' ' + txt_resum[0]]
txt_result.extend(txt_resum[1:])
# XML output
xml_result = Element('resum_rate', title=cmd_title)
xml_result.append(xml_resum)
thread_pool.join()
return PluginBase.PluginResult(txt_result, xml_result)
def _command_resum(self, target):
"""
Tests the server for session resumption support using session IDs and
TLS session tickets (RFC 5077).
"""
NB_THREADS = 5
MAX_RESUM = 5
thread_pool = ThreadPool()
for _ in xrange(MAX_RESUM): # Test 5 resumptions with session IDs
thread_pool.add_job(
(self._resume_with_session_id, (target, ), 'session_id'))
thread_pool.start(NB_THREADS)
# Test TLS tickets support while threads are running
try:
(ticket_supported,
ticket_reason) = self._resume_with_session_ticket(target)
ticket_error = None
except Exception as e:
ticket_error = str(e.__class__.__module__) + '.' + \
str(e.__class__.__name__) + ' - ' + str(e)
# Format session ID results
(txt_resum,
xml_resum) = self._format_resum_id_results(thread_pool, MAX_RESUM)
if ticket_error:
ticket_txt = 'Error: ' + ticket_error
else:
ticket_txt = 'Supported' if ticket_supported \
else 'Not Supported - ' + ticket_reason+'.'
cmd_title = 'Session Resumption'
txt_result = [self.PLUGIN_TITLE_FORMAT.format(cmd_title)]
RESUM_FORMAT = ' {0:<27} {1}'
txt_result.append(
RESUM_FORMAT.format('With Session IDs:', txt_resum[0]))
txt_result.extend(txt_resum[1:])
txt_result.append(
RESUM_FORMAT.format('With TLS Session Tickets:', ticket_txt))
# XML output
xml_resum_ticket_attr = {}
if ticket_error:
xml_resum_ticket_attr['error'] = ticket_error
else:
xml_resum_ticket_attr['isSupported'] = str(ticket_supported)
if not ticket_supported:
xml_resum_ticket_attr['reason'] = ticket_reason
xml_resum_ticket = Element('sessionResumptionWithTLSTickets',
attrib=xml_resum_ticket_attr)
xml_result = Element('resum', title=cmd_title)
xml_result.append(xml_resum)
xml_result.append(xml_resum_ticket)
thread_pool.join()
return PluginBase.PluginResult(txt_result, xml_result)
def process_task(self, target, command, arg):
if arg == 'basic':
txt_output_generator = self._get_basic_text
elif arg == 'full':
txt_output_generator = self._get_full_text
else:
raise Exception("PluginCertInfo: Unknown command.")
(host, _, _, _) = target
thread_pool = ThreadPool()
if 'ca_file' in self._shared_settings and self._shared_settings[
'ca_file']:
AVAILABLE_TRUST_STORES[self._shared_settings['ca_file']] = (
'Custom --ca_file', 'N/A')
for (store_path, _) in AVAILABLE_TRUST_STORES.iteritems():
# Try to connect with each trust store
thread_pool.add_job((self._get_cert, (target, store_path)))
# Start processing the jobs
thread_pool.start(len(AVAILABLE_TRUST_STORES))
# Store the results as they come
x509_cert_chain = []
(verify_dict, verify_dict_error, x509_cert,
ocsp_response) = ({}, {}, None, None)
for (job, result) in thread_pool.get_result():
(_, (_, store_path)) = job
(x509_cert_chain, verify_str, ocsp_response) = result
# Store the returned verify string for each trust store
x509_cert = x509_cert_chain[
0] # First cert is always the leaf cert
store_info = AVAILABLE_TRUST_STORES[store_path]
verify_dict[store_info] = verify_str
if x509_cert is None:
# This means none of the connections were successful. Get out
for (job, exception) in thread_pool.get_error():
raise exception
# Store thread pool errors
for (job, exception) in thread_pool.get_error():
(_, (_, store_path)) = job
error_msg = str(
exception.__class__.__name__) + ' - ' + str(exception)
store_info = AVAILABLE_TRUST_STORES[store_path]
verify_dict_error[store_info] = error_msg
thread_pool.join()
# Results formatting
# Text output - certificate info
text_output = [self.PLUGIN_TITLE_FORMAT('Certificate - Content')]
text_output.extend(txt_output_generator(x509_cert))
# Text output - trust validation
text_output.extend(
['', self.PLUGIN_TITLE_FORMAT('Certificate - Trust')])
# Hostname validation
if self._shared_settings['sni']:
text_output.append(
self.FIELD_FORMAT("SNI enabled with virtual domain:",
self._shared_settings['sni']))
# TODO: Use SNI name for validation when --sni was used
host_val_dict = {
X509_NAME_MATCHES_SAN: 'OK - Subject Alternative Name matches',
X509_NAME_MATCHES_CN: 'OK - Common Name matches',
X509_NAME_MISMATCH: 'FAILED - Certificate does NOT match ' + host
}
text_output.append(
self.FIELD_FORMAT("Hostname Validation:",
host_val_dict[x509_cert.matches_hostname(host)]))
# Path validation that was successful
for ((store_name, store_version),
verify_str) in verify_dict.iteritems():
verify_txt = 'OK - Certificate is trusted' if (verify_str in 'ok') \
else 'FAILED - Certificate is NOT Trusted: ' + verify_str
# EV certs - Only Mozilla supported for now
if (verify_str in 'ok') and ('Mozilla' in store_info):
if self._is_ev_certificate(x509_cert):
verify_txt += ', Extended Validation'
text_output.append(
self.FIELD_FORMAT(
self.TRUST_FORMAT(store_name=store_name,
store_version=store_version),
verify_txt))
# Path validation that ran into errors
for ((store_name, store_version),
error_msg) in verify_dict_error.iteritems():
verify_txt = 'ERROR: ' + error_msg
text_output.append(
self.FIELD_FORMAT(
self.TRUST_FORMAT(store_name=store_name,
store_version=store_version),
verify_txt))
# Print the Common Names within the certificate chain
cns_in_cert_chain = []
for cert in x509_cert_chain:
cert_identity = self._extract_subject_cn_or_oun(cert)
cns_in_cert_chain.append(cert_identity)
text_output.append(
self.FIELD_FORMAT('Certificate Chain Received:',
str(cns_in_cert_chain)))
# Text output - OCSP stapling
text_output.extend(
['', self.PLUGIN_TITLE_FORMAT('Certificate - OCSP Stapling')])
text_output.extend(self._get_ocsp_text(ocsp_response))
# XML output
xml_output = Element(command,
argument=arg,
title='Certificate Information')
# XML output - certificate chain: always return the full certificate for each cert in the chain
cert_chain_xml = Element('certificateChain')
# First add the leaf certificate
cert_chain_xml.append(
self._format_cert_to_xml(x509_cert_chain[0], 'leaf',
self._shared_settings['sni']))
# Then add every other cert in the chain
for cert in x509_cert_chain[1:]:
cert_chain_xml.append(
self._format_cert_to_xml(cert, 'intermediate',
self._shared_settings['sni']))
xml_output.append(cert_chain_xml)
# XML output - trust
trust_validation_xml = Element('certificateValidation')
# Hostname validation
is_hostname_valid = 'False' if (x509_cert.matches_hostname(host)
== X509_NAME_MISMATCH) else 'True'
host_validation_xml = Element(
'hostnameValidation',
serverHostname=host,
certificateMatchesServerHostname=is_hostname_valid)
trust_validation_xml.append(host_validation_xml)
# Path validation - OK
for ((store_name, store_version),
verify_str) in verify_dict.iteritems():
path_attrib_xml = {
'usingTrustStore': store_name,
'trustStoreVersion': store_version,
'validationResult': verify_str
}
# EV certs - Only Mozilla supported for now
if (verify_str in 'ok') and ('Mozilla' in store_info):
path_attrib_xml['isExtendedValidationCertificate'] = str(
self._is_ev_certificate(x509_cert))
trust_validation_xml.append(
Element('pathValidation', attrib=path_attrib_xml))
# Path validation - Errors
for ((store_name, store_version),
error_msg) in verify_dict_error.iteritems():
path_attrib_xml = {
'usingTrustStore': store_name,
'trustStoreVersion': store_version,
'error': error_msg
}
trust_validation_xml.append(
Element('pathValidation', attrib=path_attrib_xml))
xml_output.append(trust_validation_xml)
# XML output - OCSP Stapling
if ocsp_response is None:
ocsp_attr_xml = {'isSupported': 'False'}
ocsp_xml = Element('ocspStapling', attrib=ocsp_attr_xml)
else:
ocsp_attr_xml = {'isSupported': 'True'}
ocsp_xml = Element('ocspStapling', attrib=ocsp_attr_xml)
ocsp_resp_attr_xml = {
'isTrustedByMozillaCAStore':
str(ocsp_response.verify(MOZILLA_STORE_PATH))
}
ocsp_resp_xmp = Element('ocspResponse', attrib=ocsp_resp_attr_xml)
for (key, value) in ocsp_response.as_dict().items():
ocsp_resp_xmp.append(_keyvalue_pair_to_xml(key, value))
ocsp_xml.append(ocsp_resp_xmp)
xml_output.append(ocsp_xml)
return PluginBase.PluginResult(text_output, xml_output)
def process_task(self, target, command, args):
MAX_THREADS = 30
if command in ['sslv2', 'sslv3', 'tlsv1', 'tlsv1_1', 'tlsv1_2']:
ssl_version = command
else:
raise Exception("PluginOpenSSLCipherSuites: Unknown command.")
# Get the list of available cipher suites for the given ssl version
ctSSL_initialize(multithreading=True)
ctx = SSL_CTX.SSL_CTX(ssl_version)
ctx.set_cipher_list('ALL:NULL:@STRENGTH')
ssl = SSL.SSL(ctx)
cipher_list = ssl.get_cipher_list()
# Create a thread pool
NB_THREADS = min(len(cipher_list),
MAX_THREADS) # One thread per cipher
thread_pool = ThreadPool()
# Scan for every available cipher suite
for cipher in cipher_list:
thread_pool.add_job(
(self._test_ciphersuite, (target, ssl_version, cipher)))
# Scan for the preferred cipher suite
thread_pool.add_job((self._pref_ciphersuite, (target, ssl_version)))
# Start processing the jobs
thread_pool.start(NB_THREADS)
result_dicts = {
'preferredCipherSuite': {},
'acceptedCipherSuites': {},
'rejectedCipherSuites': {},
'errors': {}
}
# Store the results as they come
for completed_job in thread_pool.get_result():
(job, result) = completed_job
if result is not None:
(result_type, ssl_cipher, keysize, msg) = result
(result_dicts[result_type])[ssl_cipher] = (msg, keysize)
# Store thread pool errors
for failed_job in thread_pool.get_error():
(job, exception) = failed_job
ssl_cipher = str(job[1][2])
error_msg = str(exception.__class__.__module__) + '.' \
+ str(exception.__class__.__name__) + ' - ' + str(exception)
result_dicts['errors'][ssl_cipher] = (error_msg, None)
thread_pool.join()
ctSSL_cleanup()
# Generate results
return PluginBase.PluginResult(
self._generate_txt_result(result_dicts, command),
self._generate_xml_result(result_dicts, command))
def process_task(self, target, command, args):
MAX_THREADS = 15
sslVersionDict = {
'sslv2': SSLV2,
'sslv3': SSLV3,
'tlsv1': TLSV1,
'tlsv1_1': TLSV1_1,
'tlsv1_2': TLSV1_2
}
try:
sslVersion = sslVersionDict[command]
except KeyError:
raise Exception("PluginOpenSSLCipherSuites: Unknown command.")
# Get the list of available cipher suites for the given ssl version
sslClient = SslClient(sslVersion=sslVersion)
sslClient.set_cipher_list('ALL:COMPLEMENTOFALL')
cipher_list = sslClient.get_cipher_list()
# Create a thread pool
NB_THREADS = min(len(cipher_list),
MAX_THREADS) # One thread per cipher
thread_pool = ThreadPool()
# Scan for every available cipher suite
for cipher in cipher_list:
thread_pool.add_job(
(self._test_ciphersuite, (target, sslVersion, cipher)))
# Scan for the preferred cipher suite
thread_pool.add_job((self._pref_ciphersuite, (target, sslVersion)))
# Start processing the jobs
thread_pool.start(NB_THREADS)
result_dicts = {
'preferredCipherSuite': {},
'acceptedCipherSuites': {},
'rejectedCipherSuites': {},
'errors': {}
}
# Store the results as they come
for completed_job in thread_pool.get_result():
(job, result) = completed_job
if result is not None:
(result_type, ssl_cipher, keysize, dh_infos, msg) = result
(result_dicts[result_type])[ssl_cipher] = (msg, keysize,
dh_infos)
# Store thread pool errors
for failed_job in thread_pool.get_error():
(job, exception) = failed_job
ssl_cipher = str(job[1][2])
error_msg = str(
exception.__class__.__name__) + ' - ' + str(exception)
result_dicts['errors'][ssl_cipher] = (error_msg, None, None)
thread_pool.join()
# Generate results
return PluginBase.PluginResult(
self._generate_text_output(result_dicts, command),
self._generate_xml_output(result_dicts, command))
def process_task(self, target, command, arg):
if arg == 'basic':
textFunction = self._get_basic_text
elif arg == 'full':
textFunction = self._get_full_text
else:
raise Exception("PluginCertInfo: Unknown command.")
(host, _, _, _) = target
threadPool = ThreadPool()
for (storePath, _) in AVAILABLE_TRUST_STORES.iteritems():
# Try to connect with each trust store
threadPool.add_job((self._get_cert, (target, storePath)))
# Start processing the jobs
threadPool.start(len(AVAILABLE_TRUST_STORES))
# Store the results as they come
(verifyDict, verifyDictErr, x509Cert, ocspResp) = ({}, {}, None, None)
for (job, result) in threadPool.get_result():
(_, (_, storePath)) = job
(x509Chain, verifyStr, ocspResp) = result
# Store the returned verify string for each trust store
x509Cert = x509Chain[0] # First cert is always the leaf cert
storeName = AVAILABLE_TRUST_STORES[storePath]
verifyDict[storeName] = verifyStr
if x509Cert is None:
# This means none of the connections were successful. Get out
for (job, exception) in threadPool.get_error():
raise exception
# Store thread pool errors
for (job, exception) in threadPool.get_error():
(_, (_, storePath)) = job
errorMsg = str(exception.__class__.__name__) + ' - ' \
+ str(exception)
storeName = AVAILABLE_TRUST_STORES[storePath]
verifyDictErr[storeName] = errorMsg
threadPool.join()
# Results formatting
# Text output - certificate info
outputTxt = [self.PLUGIN_TITLE_FORMAT('Certificate - Content')]
outputTxt.extend(textFunction(x509Cert))
# Text output - trust validation
outputTxt.extend(['', self.PLUGIN_TITLE_FORMAT('Certificate - Trust')])
# Hostname validation
if self._shared_settings['sni']:
outputTxt.append(
self.FIELD_FORMAT("SNI enabled with virtual domain:",
self._shared_settings['sni']))
# TODO: Use SNI name for validation when --sni was used
hostValDict = {
X509_NAME_MATCHES_SAN: 'OK - Subject Alternative Name matches',
X509_NAME_MATCHES_CN: 'OK - Common Name matches',
X509_NAME_MISMATCH: 'FAILED - Certificate does NOT match ' + host
}
outputTxt.append(
self.FIELD_FORMAT("Hostname Validation:",
hostValDict[x509Cert.matches_hostname(host)]))
# Path validation that was successful
for (storeName, verifyStr) in verifyDict.iteritems():
verifyTxt = 'OK - Certificate is trusted' if (
verifyStr in 'ok'
) else 'FAILED - Certificate is NOT Trusted: ' + verifyStr
# EV certs - Only Mozilla supported for now
if (verifyStr in 'ok') and ('Mozilla' in storeName):
if (self._is_ev_certificate(x509Cert)):
verifyTxt += ', Extended Validation'
outputTxt.append(
self.FIELD_FORMAT(self.TRUST_FORMAT(storeName), verifyTxt))
# Path validation that ran into errors
for (storeName, errorMsg) in verifyDictErr.iteritems():
verifyTxt = 'ERROR: ' + errorMsg
outputTxt.append(
self.FIELD_FORMAT(self.TRUST_FORMAT(storeName), verifyTxt))
# Print the Common Names within the certificate chain
certChainCNs = []
for cert in x509Chain:
certIdentity = self._extract_subject_CN_or_OUN(cert)
certChainCNs.append(certIdentity)
outputTxt.append(
self.FIELD_FORMAT('Certificate Chain Received:',
str(certChainCNs)))
# Text output - OCSP stapling
outputTxt.extend(
['', self.PLUGIN_TITLE_FORMAT('Certificate - OCSP Stapling')])
outputTxt.extend(self._get_ocsp_text(ocspResp))
# XML output
outputXml = Element(command,
argument=arg,
title='Certificate Information')
# XML output - certificate chain: always return the full certificate for each cert in the chain
chainXml = Element('certificateChain')
# First add the leaf certificate
chainXml.append(
self._format_cert_to_xml(x509Chain[0], 'leaf',
self._shared_settings['sni']))
# Then add every other cert in the chain
for cert in x509Chain[1:]:
chainXml.append(
self._format_cert_to_xml(cert, 'intermediate',
self._shared_settings['sni']))
outputXml.append(chainXml)
# XML output - trust
trustXml = Element('certificateValidation')
# Hostname validation
hostValBool = 'False' if (x509Cert.matches_hostname(host) == X509_NAME_MISMATCH) \
else 'True'
hostXml = Element('hostnameValidation',
serverHostname=host,
certificateMatchesServerHostname=hostValBool)
trustXml.append(hostXml)
# Path validation - OK
for (storeName, verifyStr) in verifyDict.iteritems():
pathXmlAttrib = {
'usingTrustStore': storeName,
'validationResult': verifyStr
}
# EV certs - Only Mozilla supported for now
if (verifyStr in 'ok') and ('Mozilla' in storeName):
pathXmlAttrib['isExtendedValidationCertificate'] = str(
self._is_ev_certificate(x509Cert))
trustXml.append(Element('pathValidation', attrib=pathXmlAttrib))
# Path validation - Errors
for (storeName, errorMsg) in verifyDictErr.iteritems():
pathXmlAttrib = {'usingTrustStore': storeName, 'error': errorMsg}
trustXml.append(Element('pathValidation', attrib=pathXmlAttrib))
outputXml.append(trustXml)
# XML output - OCSP Stapling
if ocspResp is None:
oscpAttr = {'error': 'Server did not send back an OCSP response'}
ocspXml = Element('ocspStapling', attrib=oscpAttr)
else:
oscpAttr = {
'isTrustedByMozillaCAStore':
str(ocspResp.verify(MOZILLA_STORE_PATH))
}
ocspXml = Element('ocspResponse', attrib=oscpAttr)
for (key, value) in ocspResp.as_dict().items():
ocspXml.append(_keyvalue_pair_to_xml(key, value))
outputXml.append(ocspXml)
return PluginBase.PluginResult(outputTxt, outputXml)
def process_task(self, target, command, args):
if 'algorithm' in self._shared_settings and self._shared_settings[
'algorithm']:
algorithm = self._shared_settings['algorithm']
else:
algorithm = 'naive'
logging.info("SUBSTART\t%s\t%s\t%s\t%s\t%s" %
(datetime.utcnow(), self._shared_settings['targets_in'],
target[0], command, algorithm))
MAX_THREADS = 15
sslVersionDict = {
'sslv2': SSLV2,
'sslv3': SSLV3,
'tlsv1': TLSV1,
'tlsv1_1': TLSV1_1,
'tlsv1_2': TLSV1_2
}
result_dicts = {
'preferredCipherSuite': {},
'acceptedCipherSuites': {},
'rejectedCipherSuites': {},
'errors': {}
}
try:
sslVersion = sslVersionDict[command]
except KeyError:
raise Exception("PluginOpenSSLCipherSuites: Unknown command.")
# Get the list of available cipher suites for the given ssl version
sslClient = SslClient(sslVersion=sslVersion)
sslClient.set_cipher_list('ALL:COMPLEMENTOFALL')
cipher_list = sslClient.get_cipher_list()
NB_THREADS = min(len(cipher_list),
MAX_THREADS) # One thread per cipher
# Create a thread pool
thread_pool = ThreadPool()
# First add the "pref" job to only execute it once
# Scan for the preferred cipher suite
if algorithm != 'connopt':
thread_pool.add_job((self._pref_ciphersuite, (target, sslVersion)))
log_round_counter = 0
while (len(result_dicts['acceptedCipherSuites']) +
len(result_dicts['rejectedCipherSuites']) +
len(result_dicts['errors']) < len(cipher_list)):
log_round_counter += 1
new_jobs = self._calculate_jobs(sslVersion, cipher_list,
result_dicts, algorithm, target[2])
for job in new_jobs:
thread_pool.add_job(
(self._test_ciphersuite, (target, sslVersion, job)))
# logging.debug("Adding following jobs:\n%s" % pprint.pformat(new_jobs))
# logging.debug("%s: round=%d, new_jobs=%d, algorithm=%s" % (sslVersion,
# log_round_counter,
# len(new_jobs),
# algorithm))
# Start processing the jobs
thread_pool.start(NB_THREADS)
# Store the results as they come
for completed_job in thread_pool.get_result():
(job, results) = completed_job
for result in results:
(result_type, ssl_cipher, keysize, dh_infos, msg) = result
(result_dicts[result_type])[ssl_cipher] = (msg, keysize,
dh_infos)
# Store thread pool errors
for failed_job in thread_pool.get_error():
(job, exception) = failed_job
# job[1][2] is a list of cipher suites now
ssl_ciphers = job[1][2]
error_msg = str(
exception.__class__.__name__) + ' - ' + str(exception)
for ssl_cipher in ssl_ciphers:
result_dicts['errors'][ssl_cipher] = (error_msg, None,
None)
thread_pool.join()
# Reset thread pool
thread_pool = ThreadPool()
# logging.debug("ciphers total %d results a: %d, r: %d, e: %d after %d connections" % (
# len(cipher_list),
# len(result_dicts['acceptedCipherSuites']),
# len(result_dicts['rejectedCipherSuites']),
# len(result_dicts['errors']),
# self.log_connection_counter))
timedelta = datetime.now() - self.log_starttime
logging.info("RESULT\t%s\t%s\t%s" % (target[0], command, ",".join(
stats.get_pattern_for_result_dict(sslVersion, result_dicts))))
logging.info("SUBEND\t%s\t%s\t%s\t%s\t%s\t%s\t%s" %
(datetime.utcnow(), self._shared_settings['targets_in'],
target[0], command, algorithm, timedelta.total_seconds(),
self.log_connection_counter))
increment(self.log_connection_counter)
# Generate results
return PluginBase.PluginResult(
self._generate_text_output(result_dicts, command),
self._generate_xml_output(result_dicts, command))
