def test_with_empty(self): ret = client_cert_types_to_ids("") self.assertEqual(ret, [])
def test_with_one(self): ret = client_cert_types_to_ids("rsa_sign") self.assertEqual(ret, [1])
def test_with_unknown_name(self): with self.assertRaises(AttributeError): client_cert_types_to_ids("ed448_sign")
def test_with_malformed_integer(self): with self.assertRaises(AttributeError): client_cert_types_to_ids("1/23 ecdsa_sign")
def test_with_mixed(self): ret = client_cert_types_to_ids("1 ecdsa_sign") self.assertEqual(ret, [1, 64])
def main(): """Check what signature algorithms server advertises""" hostname = "localhost" port = 4433 run_exclude = set() cert = None private_key = None sigalgs = [ SignatureScheme.ecdsa_secp521r1_sha512, SignatureScheme.ecdsa_secp384r1_sha384, SignatureScheme.ecdsa_secp256r1_sha256, (HashAlgorithm.sha224, SignatureAlgorithm.ecdsa), (HashAlgorithm.sha1, SignatureAlgorithm.ecdsa), SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha512, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_pss_sha256, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa) ] cert_types = [ ClientCertificateType.rsa_sign, ClientCertificateType.ecdsa_sign ] argv = sys.argv[1:] opts, args = getopt.getopt(argv, "h:p:e:s:k:c:T:", ["help"]) for opt, arg in opts: if opt == '-h': host = arg elif opt == '-p': port = int(arg) elif opt == '-e': run_exclude.add(arg) elif opt == '--help': help_msg() sys.exit(0) elif opt == '-s': sigalgs = sig_algs_to_ids(arg) elif opt == '-T': cert_types = client_cert_types_to_ids(arg) elif opt == '-k': text_key = open(arg, 'rb').read() if sys.version_info[0] >= 3: text_key = str(text_key, 'utf-8') private_key = parsePEMKey(text_key, private=True) elif opt == '-c': text_cert = open(arg, 'rb').read() if sys.version_info[0] >= 3: text_cert = str(text_cert, 'utf-8') cert = X509() cert.parse(text_cert) else: raise ValueError("Unknown option: {0}".format(opt)) if args: run_only = set(args) else: run_only = None conversations = {} # sanity check for Client Certificates conversation = Connect(hostname, port) node = conversation ciphers = [ CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV ] sigs = [ SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha256, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_pss_sha512, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa), (HashAlgorithm.md5, SignatureAlgorithm.rsa) ] ext = { ExtensionType.signature_algorithms: SignatureAlgorithmsExtension().create(sigs), ExtensionType.signature_algorithms_cert: SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL) } node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello(version=(3, 3))) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectCertificateRequest()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(CertificateGenerator()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator(b"GET / HTTP/1.0\n\n")) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertDescription.close_notify)) node = node.add_child(ExpectClose()) node.next_sibling = ExpectAlert() node.next_sibling.add_child(ExpectClose()) conversations["sanity"] = conversation if cert and private_key: # sanity check for Client Certificates conversation = Connect(hostname, port) node = conversation ciphers = [ CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV ] sigs = [ SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha256, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_pss_sha512, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa), (HashAlgorithm.md5, SignatureAlgorithm.rsa) ] ext = { ExtensionType.signature_algorithms: SignatureAlgorithmsExtension().create(sigs), ExtensionType.signature_algorithms_cert: SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL) } node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello(version=(3, 3))) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectCertificateRequest()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(CertificateGenerator(X509CertChain([cert]))) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(CertificateVerifyGenerator(private_key)) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator(b"GET / HTTP/1.0\n\n")) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertDescription.close_notify)) node = node.add_child(ExpectClose()) node.next_sibling = ExpectAlert() node.next_sibling.add_child(ExpectClose()) conversations["with certificate"] = conversation # verify the advertised hashes conversation = Connect(hostname, port) node = conversation ciphers = [ CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV ] sigs = [ SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha256, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_pss_sha512, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa), (HashAlgorithm.md5, SignatureAlgorithm.rsa) ] ext = { ExtensionType.signature_algorithms: SignatureAlgorithmsExtension().create(sigs), ExtensionType.signature_algorithms_cert: SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL) } node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello(version=(3, 3))) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectCertificateRequest(sigalgs)) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(CertificateGenerator()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator(b"GET / HTTP/1.0\n\n")) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertDescription.close_notify)) node = node.add_child(ExpectClose()) node.next_sibling = ExpectAlert() node.next_sibling.add_child(ExpectClose()) conversations["check sigalgs in cert request"] = conversation # verify the advertised certificate types conversation = Connect(hostname, port) node = conversation ciphers = [ CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV ] sigs = [ SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha256, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_pss_sha512, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa), (HashAlgorithm.md5, SignatureAlgorithm.rsa) ] ext = { ExtensionType.signature_algorithms: SignatureAlgorithmsExtension().create(sigs), ExtensionType.signature_algorithms_cert: SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL) } node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello(version=(3, 3))) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectCertificateRequest(cert_types=cert_types)) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(CertificateGenerator()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator(b"GET / HTTP/1.0\n\n")) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertDescription.close_notify)) node = node.add_child(ExpectClose()) node.next_sibling = ExpectAlert() node.next_sibling.add_child(ExpectClose()) conversations["check cert types in cert request"] = conversation # run the conversation good = 0 bad = 0 failed = [] # make sure that sanity test is run first and last # to verify that server was running and kept running throughout sanity_tests = [('sanity', conversations['sanity'])] regular_tests = [(k, v) for k, v in conversations.items() if k != 'sanity'] shuffled_tests = sample(regular_tests, len(regular_tests)) ordered_tests = chain(sanity_tests, shuffled_tests, sanity_tests) for c_name, c_test in ordered_tests: if run_only and c_name not in run_only or c_name in run_exclude: continue print("{0} ...".format(c_name)) runner = Runner(c_test) res = True try: runner.run() except: print("Error while processing") print(traceback.format_exc()) res = False if res: good += 1 print("OK\n") else: bad += 1 failed.append(c_name) print("Test to verify if server accepts empty certificate messages and") print("advertises only expected signature algotithms in Certificate") print("Request message\n") print("version: {0}\n".format(version)) print("Test end") print("successful: {0}".format(good)) print("failed: {0}".format(bad)) failed_sorted = sorted(failed, key=natural_sort_keys) print(" {0}".format('\n '.join(repr(i) for i in failed_sorted))) if bad > 0: sys.exit(1)
def main(): """Check what signature algorithms server advertises""" hostname = "localhost" port = 4433 num_limit = None run_exclude = set() expected_failures = {} last_exp_tmp = None cert = None private_key = None sigalgs = [SignatureScheme.ed25519, SignatureScheme.ed448, SignatureScheme.ecdsa_secp521r1_sha512, SignatureScheme.ecdsa_secp384r1_sha384, SignatureScheme.ecdsa_secp256r1_sha256, (HashAlgorithm.sha224, SignatureAlgorithm.ecdsa), (HashAlgorithm.sha1, SignatureAlgorithm.ecdsa), SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha512, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_pss_sha256, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa)] cert_types = [ClientCertificateType.rsa_sign, ClientCertificateType.ecdsa_sign] argv = sys.argv[1:] opts, args = getopt.getopt(argv, "h:p:e:x:X:s:k:c:T:", ["help"]) for opt, arg in opts: if opt == '-h': hostname = arg elif opt == '-p': port = int(arg) elif opt == '-e': run_exclude.add(arg) elif opt == '-n': num_limit = int(arg) elif opt == '-x': expected_failures[arg] = None last_exp_tmp = str(arg) elif opt == '-X': if not last_exp_tmp: raise ValueError("-x has to be specified before -X") expected_failures[last_exp_tmp] = str(arg) elif opt == '--help': help_msg() sys.exit(0) elif opt == '-s': sigalgs = sig_algs_to_ids(arg) elif opt == '-T': cert_types = client_cert_types_to_ids(arg) elif opt == '-k': text_key = open(arg, 'rb').read() if sys.version_info[0] >= 3: text_key = str(text_key, 'utf-8') private_key = parsePEMKey(text_key, private=True) elif opt == '-c': text_cert = open(arg, 'rb').read() if sys.version_info[0] >= 3: text_cert = str(text_cert, 'utf-8') cert = X509() cert.parse(text_cert) else: raise ValueError("Unknown option: {0}".format(opt)) if args: run_only = set(args) else: run_only = None conversations = {} # sanity check for Client Certificates conversation = Connect(hostname, port) node = conversation ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] sigs = [SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha256, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_pss_sha512, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa), (HashAlgorithm.md5, SignatureAlgorithm.rsa)] ext = {ExtensionType.signature_algorithms : SignatureAlgorithmsExtension().create(sigs), ExtensionType.signature_algorithms_cert : SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL)} node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello(version=(3, 3))) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectCertificateRequest()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(CertificateGenerator()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator(b"GET / HTTP/1.0\n\n")) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertDescription.close_notify)) node = node.add_child(ExpectClose()) node.next_sibling = ExpectAlert() node.next_sibling.add_child(ExpectClose()) conversations["sanity"] = conversation if cert and private_key: # sanity check for Client Certificates conversation = Connect(hostname, port) node = conversation ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] sigs = [SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha256, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_pss_sha512, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa), (HashAlgorithm.md5, SignatureAlgorithm.rsa)] ext = {ExtensionType.signature_algorithms : SignatureAlgorithmsExtension().create(sigs), ExtensionType.signature_algorithms_cert : SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL)} node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello(version=(3, 3))) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectCertificateRequest()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(CertificateGenerator(X509CertChain([cert]))) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(CertificateVerifyGenerator(private_key)) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator(b"GET / HTTP/1.0\n\n")) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertDescription.close_notify)) node = node.add_child(ExpectClose()) node.next_sibling = ExpectAlert() node.next_sibling.add_child(ExpectClose()) conversations["with certificate"] = conversation # verify the advertised hashes conversation = Connect(hostname, port) node = conversation ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] sigs = [SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha256, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_pss_sha512, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa), (HashAlgorithm.md5, SignatureAlgorithm.rsa)] ext = {ExtensionType.signature_algorithms : SignatureAlgorithmsExtension().create(sigs), ExtensionType.signature_algorithms_cert : SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL)} node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello(version=(3, 3))) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectCertificateRequest(sigalgs)) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(CertificateGenerator()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator(b"GET / HTTP/1.0\n\n")) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertDescription.close_notify)) node = node.add_child(ExpectClose()) node.next_sibling = ExpectAlert() node.next_sibling.add_child(ExpectClose()) conversations["check sigalgs in cert request"] = conversation # verify the advertised certificate types conversation = Connect(hostname, port) node = conversation ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] sigs = [SignatureScheme.rsa_pss_rsae_sha256, SignatureScheme.rsa_pss_rsae_sha384, SignatureScheme.rsa_pss_rsae_sha512, SignatureScheme.rsa_pss_pss_sha256, SignatureScheme.rsa_pss_pss_sha384, SignatureScheme.rsa_pss_pss_sha512, (HashAlgorithm.sha512, SignatureAlgorithm.rsa), (HashAlgorithm.sha384, SignatureAlgorithm.rsa), (HashAlgorithm.sha256, SignatureAlgorithm.rsa), (HashAlgorithm.sha224, SignatureAlgorithm.rsa), (HashAlgorithm.sha1, SignatureAlgorithm.rsa), (HashAlgorithm.md5, SignatureAlgorithm.rsa)] ext = {ExtensionType.signature_algorithms : SignatureAlgorithmsExtension().create(sigs), ExtensionType.signature_algorithms_cert : SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL)} node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello(version=(3, 3))) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectCertificateRequest(cert_types=cert_types)) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(CertificateGenerator()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator(b"GET / HTTP/1.0\n\n")) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertDescription.close_notify)) node = node.add_child(ExpectClose()) node.next_sibling = ExpectAlert() node.next_sibling.add_child(ExpectClose()) conversations["check cert types in cert request"] = conversation # run the conversation good = 0 bad = 0 xfail = 0 xpass = 0 failed = [] xpassed = [] if not num_limit: num_limit = len(conversations) # make sure that sanity test is run first and last # to verify that server was running and kept running throughout sanity_tests = [('sanity', conversations['sanity'])] if run_only: if num_limit > len(run_only): num_limit = len(run_only) regular_tests = [(k, v) for k, v in conversations.items() if k in run_only] else: regular_tests = [(k, v) for k, v in conversations.items() if (k != 'sanity') and k not in run_exclude] sampled_tests = sample(regular_tests, min(num_limit, len(regular_tests))) ordered_tests = chain(sanity_tests, sampled_tests, sanity_tests) for c_name, c_test in ordered_tests: if run_only and c_name not in run_only or c_name in run_exclude: continue print("{0} ...".format(c_name)) runner = Runner(c_test) res = True exception = None try: runner.run() except Exception as exp: exception = exp print("Error while processing") print(traceback.format_exc()) res = False if c_name in expected_failures: if res: xpass += 1 xpassed.append(c_name) print("XPASS-expected failure but test passed\n") else: if expected_failures[c_name] is not None and \ expected_failures[c_name] not in str(exception): bad += 1 failed.append(c_name) print("Expected error message: {0}\n" .format(expected_failures[c_name])) else: xfail += 1 print("OK-expected failure\n") else: if res: good += 1 print("OK\n") else: bad += 1 failed.append(c_name) print("Test to verify if server accepts empty certificate messages and") print("advertises only expected signature algotithms in Certificate") print("Request message\n") print("Test end") print(20 * '=') print("version: {0}".format(version)) print(20 * '=') print("TOTAL: {0}".format(len(sampled_tests) + 2*len(sanity_tests))) print("SKIP: {0}".format(len(run_exclude.intersection(conversations.keys())))) print("PASS: {0}".format(good)) print("XFAIL: {0}".format(xfail)) print("FAIL: {0}".format(bad)) print("XPASS: {0}".format(xpass)) print(20 * '=') sort = sorted(xpassed ,key=natural_sort_keys) if len(sort): print("XPASSED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort))) sort = sorted(failed, key=natural_sort_keys) if len(sort): print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort))) if bad or xpass: sys.exit(1)