def scan_certs_by_subject(self, subject_name, limit=0):
"""Scan certificates matching a subject name.
Args:
subject_name: a subject name, usually a domain. A scan for
example.com returns certificates for www.example.com,
*.example.com, test.mail.example.com, etc. Similarly
'com' can be used to look for all .com certificates.
Wildcards are treated as literal characters: a search
for *.example.com returns certificates for
*.example.com but not for mail.example.com and vice
versa.
Name may also be a common name rather than a DNS name,
e.g., "Trustworthy Certificate Authority".
limit: maximum number of entries to yield. Default is no
limit.
Yields:
DER-encoded certificates."""
sql_limit = -1 if not limit else limit
prefix = cert_desc.process_name(subject_name)
with self.__mgr.get_connection() as conn:
for row in conn.execute(
"SELECT certs.cert as cert, subject_names.name as name "
"FROM certs, subject_names WHERE name >= ? AND certs.id == "
"subject_names.cert_id ORDER BY name ASC LIMIT ?",
(".".join(prefix), sql_limit)):
name = cert_desc.process_name(row["name"], reverse=False)
if self.__compare_processed_names(prefix, name):
yield str(row["cert"])
else:
break
def scan_certs_by_subject(self, subject_name, limit=0):
"""Scan certificates matching a subject name.
Args:
subject_name: a subject name, usually a domain. A scan for
example.com returns certificates for www.example.com,
*.example.com, test.mail.example.com, etc. Similarly
'com' can be used to look for all .com certificates.
Wildcards are treated as literal characters: a search
for *.example.com returns certificates for
*.example.com but not for mail.example.com and vice
versa.
Name may also be a common name rather than a DNS name,
e.g., "Trustworthy Certificate Authority".
limit: maximum number of entries to yield. Default is no
limit.
Yields:
DER-encoded certificates."""
sql_limit = -1 if not limit else limit
prefix = cert_desc.process_name(subject_name)
with self.__mgr.get_connection() as conn:
for row in conn.execute(
"SELECT certs.cert as cert, subject_names.name as name "
"FROM certs, subject_names WHERE name >= ? AND certs.id == "
"subject_names.cert_id ORDER BY name ASC LIMIT ?",
(".".join(prefix), sql_limit)):
name = cert_desc.process_name(row["name"], reverse=False)
if self.__compare_processed_names(prefix, name):
yield str(row["cert"])
else:
break
def test_from_cert(self):
observations = []
for check in all_checks.ALL_CHECKS:
observations += check.check(CERT) or []
observations.append(
observation.Observation("AE", u'ćę©ß→æ→ćąßę-ß©ąńśþa©ęńć←',
(u'əꔹłęµ', u'…łą↓ð→↓ś→ę')))
proto = cert_desc.from_cert(CERT, observations)
self.assertEqual(proto.der, CERT.to_der())
subject = [(att.type, att.value) for att in proto.subject]
cert_subject = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in CERT.subject()]
self.assertItemsEqual(cert_subject, subject)
issuer = [(att.type, att.value) for att in proto.issuer]
cert_issuer = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in CERT.issuer()]
self.assertItemsEqual(cert_issuer, issuer)
subject_alternative_names = [
(att.type, att.value) for att in proto.subject_alternative_names
]
cert_subject_alternative_names = [
(san.component_key(),
cert_desc.to_unicode('.'.join(
cert_desc.process_name(
san.component_value().human_readable()))))
for san in CERT.subject_alternative_names()
]
self.assertItemsEqual(cert_subject_alternative_names,
subject_alternative_names)
self.assertEqual(proto.version, str(CERT.version().value))
self.assertEqual(
proto.serial_number,
str(CERT.serial_number().human_readable().upper().replace(':',
'')))
self.assertEqual(time.gmtime(proto.validity.not_before / 1000),
CERT.not_before())
self.assertEqual(time.gmtime(proto.validity.not_after / 1000),
CERT.not_after())
observations_tuples = [(unicode(obs.description),
unicode(obs.reason) if obs.reason else u'',
obs.details_to_proto())
for obs in observations]
proto_obs = [(obs.description, obs.reason, obs.details)
for obs in proto.observations]
self.assertItemsEqual(proto_obs, observations_tuples)
def test_from_cert(self):
desc = cert_desc.CertificateDescription.from_cert(CERT)
self.assertEqual(desc.der, CERT.to_der())
self.assertEqual(desc.subject_names,
['.'.join(cert_desc.process_name(sub.value))
for sub in CERT.subject_common_names()])
self.assertEqual(desc.alt_subject_names,
['.'.join(cert_desc.process_name(sub.value))
for sub in CERT.subject_dns_names()])
self.assertEqual(desc.version, str(CERT.version().value))
self.assertEqual(desc.serial_number, str(CERT.serial_number().value))
self.assertEqual(desc.ip_addresses,
[str(ip) for ip in CERT.subject_ip_addresses()])
def test_from_cert(self):
observations = []
for check in all_checks.ALL_CHECKS:
observations += check.check(CERT) or []
observations.append(observation.Observation(
"AE", u'ćę©ß→æ→ćąßę-ß©ąńśþa©ęńć←', (u'əꔹłęµ', u'…łą↓ð→↓ś→ę')))
proto = cert_desc.from_cert(CERT, observations)
self.assertEqual(proto.der, CERT.to_der())
subject = [(att.type, att.value) for att in proto.subject]
cert_subject = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in CERT.subject()]
self.assertItemsEqual(cert_subject, subject)
issuer = [(att.type, att.value) for att in proto.issuer]
cert_issuer = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in CERT.issuer()]
self.assertItemsEqual(cert_issuer, issuer)
subject_alternative_names = [(att.type, att.value)
for att in proto.subject_alternative_names]
cert_subject_alternative_names = [(san.component_key(),
cert_desc.to_unicode('.'.join(
cert_desc.process_name(
san.component_value().human_readable()))))
for san in CERT.subject_alternative_names()]
self.assertItemsEqual(cert_subject_alternative_names,
subject_alternative_names)
self.assertEqual(proto.version, str(CERT.version().value))
self.assertEqual(proto.serial_number,
str(CERT.serial_number().human_readable()
.upper().replace(':', '')))
self.assertEqual(time.gmtime(proto.validity.not_before / 1000),
CERT.not_before())
self.assertEqual(time.gmtime(proto.validity.not_after / 1000),
CERT.not_after())
observations_tuples = [(unicode(obs.description),
unicode(obs.reason) if obs.reason else u'',
obs.details_to_proto())
for obs in observations]
proto_obs = [(obs.description, obs.reason, obs.details)
for obs in proto.observations]
self.assertItemsEqual(proto_obs, observations_tuples)
def assert_description_subject_matches_source(self, proto, source):
subject = [(att.type, att.value) for att in proto.subject]
cert_subject = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in source.subject()]
self.assertItemsEqual(cert_subject, subject)
def assert_description_issuer_matches_source(self, proto, source):
issuer = [(att.type, att.value) for att in proto.issuer]
cert_issuer = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in source.issuer()]
self.assertItemsEqual(cert_issuer, issuer)
def assert_description_subject_matches_source(self, proto, source):
subject = [(att.type, att.value) for att in proto.subject]
cert_subject = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in source.subject()]
self.assertItemsEqual(cert_subject, subject)
def assert_description_issuer_matches_source(self, proto, source):
issuer = [(att.type, att.value) for att in proto.issuer]
cert_issuer = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in source.issuer()]
self.assertItemsEqual(cert_issuer, issuer)
def _scan_der_cert(der_certs):
current = -1
result = []
for log_index, der_cert, der_chain, entry_type in der_certs:
try:
current = log_index
certificate = None
strict_failure = False
try:
certificate = cert.Certificate(der_cert)
except error.Error as e:
try:
certificate = cert.Certificate(der_cert, strict_der=False)
except error.Error as e:
strict_failure = True
if not strict_failure:
desc = cert_desc.from_cert(certificate)
else:
desc = certificate_pb2.X509Description()
desc.der = der_cert
desc.sha256_hash = hashlib.sha256(der_cert).digest()
desc.entry_type = entry_type
root = None
if der_chain:
try:
issuer = cert.Certificate(der_chain[0], strict_der=False)
except error.Error:
pass
else:
desc.issuer_pk_sha256_hash = issuer.key_hash(hashfunc="sha256")
try:
root = cert.Certificate(der_chain[-1], strict_der=False)
except error.Error:
pass
else:
# No chain implies this is a root certificate.
# Note that certificate may be None.
root = certificate
if root:
for iss in [(type_.short_name, cert_desc.to_unicode(
'.'.join(cert_desc.process_name(value.human_readable()))))
for type_, value in root.issuer()]:
proto_iss = desc.root_issuer.add()
proto_iss.type, proto_iss.value = iss
result.append((desc, log_index))
except:
batch_start_index, batch_end_index = (
der_certs[0][0], der_certs[-1][0])
logging.exception(
"Error scanning certificate %d in batch <%d, %d> - it will "
"be excluded from the scan results",
current, batch_start_index, batch_end_index)
return result
def assert_description_alt_subject_names_match_source(self, proto, source):
subject_alternative_names = [(att.type, att.value)
for att in proto.subject_alternative_names]
cert_subject_alternative_names = [(san.component_key(),
cert_desc.to_unicode('.'.join(
cert_desc.process_name(
san.component_value().human_readable()))))
for san in source.subject_alternative_names()]
self.assertItemsEqual(cert_subject_alternative_names,
subject_alternative_names)
def assert_description_alt_subject_names_match_source(self, proto, source):
subject_alternative_names = [(att.type, att.value)
for att in proto.subject_alternative_names]
cert_subject_alternative_names = [(san.component_key(),
cert_desc.to_unicode('.'.join(
cert_desc.process_name(
san.component_value().human_readable()))))
for san in source.subject_alternative_names()]
self.assertItemsEqual(cert_subject_alternative_names,
subject_alternative_names)
def _scan_der_cert(der_certs, checks):
current = -1
try:
result = []
for log_index, der_cert, der_chain in der_certs:
current = log_index
partial_result = []
strict_failure = False
try:
certificate = cert.Certificate(der_cert)
except error.Error as e:
try:
certificate = cert.Certificate(der_cert, strict_der=False)
except error.Error as e:
partial_result.append(asn1.All())
strict_failure = True
else:
if isinstance(e, error.ASN1IllegalCharacter):
partial_result.append(
asn1.Strict(reason=e.args[0],
details=(e.string, e.index)))
else:
partial_result.append(asn1.Strict(reason=str(e)))
if not strict_failure:
for check in checks:
partial_result += check.check(certificate) or []
desc = cert_desc.from_cert(certificate, partial_result)
else:
desc = certificate_pb2.X509Description()
desc.der = der_cert
desc.sha256_hash = hashlib.sha256(der_cert).digest()
try:
root = cert.Certificate(der_chain[-1], strict_der=False)
except error.Error:
pass
else:
for iss in [
(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in root.issuer()
]:
proto_iss = desc.root_issuer.add()
proto_iss.type, proto_iss.value = iss
result.append((desc, log_index, partial_result))
return result
except Exception:
_, exception, exception_traceback = sys.exc_info()
exception_traceback = traceback.format_exc(exception_traceback)
raise PoolException((exception, exception_traceback, der_certs[0][0],
der_certs[-1][0], current))
def _scan_der_cert(der_certs, checks):
current = -1
try:
result = []
for log_index, der_cert, der_chain, entry_type in der_certs:
current = log_index
partial_result = []
strict_failure = False
try:
certificate = cert.Certificate(der_cert)
except error.Error as e:
try:
certificate = cert.Certificate(der_cert, strict_der=False)
except error.Error as e:
partial_result.append(asn1.All())
strict_failure = True
else:
if isinstance(e, error.ASN1IllegalCharacter):
partial_result.append(asn1.Strict(reason=e.args[0],
details=(e.string, e.index)))
else:
partial_result.append(asn1.Strict(reason=str(e)))
if not strict_failure:
for check in checks:
partial_result += check.check(certificate) or []
desc = cert_desc.from_cert(certificate, partial_result)
else:
desc = certificate_pb2.X509Description()
desc.der = der_cert
desc.sha256_hash = hashlib.sha256(der_cert).digest()
desc.entry_type = entry_type
try:
root = cert.Certificate(der_chain[-1], strict_der=False)
except error.Error:
pass
else:
for iss in [(type_.short_name, cert_desc.to_unicode(
'.'.join(cert_desc.process_name(value.human_readable()))))
for type_, value in root.issuer()]:
proto_iss = desc.root_issuer.add()
proto_iss.type, proto_iss.value = iss
result.append((desc, log_index, partial_result))
return result
except Exception:
_, exception, exception_traceback = sys.exc_info()
exception_traceback = traceback.format_exc(exception_traceback)
raise PoolException((exception, exception_traceback,
der_certs[0][0], der_certs[-1][0], current))
def test_process_value(self):
self.assertEqual(['London'], cert_desc.process_name('London'))
self.assertEqual(['Bob Smith'], cert_desc.process_name('Bob Smith'))
self.assertEqual(['com', 'googleapis', 'ct'], cert_desc.process_name('ct.googleapis.com'))
self.assertEqual(['com', 'github'], cert_desc.process_name('gItHuB.CoM'))
# These two are unfortunate outcomes:
# 1. single-word hostnames are indistinguishable from single-word CN
# terms like State, City, Organization
self.assertEqual(['LOCALhost'], cert_desc.process_name('LOCALhost'))
# 2. IP addresses should perhaps not be reversed like hostnames are
self.assertEqual(['1', '0', '168', '192'], cert_desc.process_name('192.168.0.1'))
def test_process_value(self):
self.assertEqual(["London"], cert_desc.process_name("London"))
self.assertEqual(["Bob Smith"], cert_desc.process_name("Bob Smith"))
self.assertEqual(["com", "googleapis", "ct"],
cert_desc.process_name("ct.googleapis.com"))
self.assertEqual(["com", "github"],
cert_desc.process_name("gItHuB.CoM"))
# These two are unfortunate outcomes:
# 1. single-word hostnames are indistinguishable from single-word CN
# terms like State, City, Organization
self.assertEqual(["LOCALhost"], cert_desc.process_name("LOCALhost"))
# 2. IP addresses should perhaps not be reversed like hostnames are
self.assertEqual(["1", "0", "168", "192"],
cert_desc.process_name("192.168.0.1"))
def test_process_value(self):
self.assertEqual(["London"], cert_desc.process_name("London"))
self.assertEqual(["Bob Smith"], cert_desc.process_name("Bob Smith"))
self.assertEqual(["com", "googleapis", "ct"],
cert_desc.process_name("ct.googleapis.com"))
self.assertEqual(["com", "github"],
cert_desc.process_name("gItHuB.CoM"))
# These two are unfortunate outcomes:
# 1. single-word hostnames are indistinguishable from single-word CN
# terms like State, City, Organization
self.assertEqual(["LOCALhost"], cert_desc.process_name("LOCALhost"))
# 2. IP addresses should perhaps not be reversed like hostnames are
self.assertEqual(["1", "0", "168", "192"],
cert_desc.process_name("192.168.0.1"))
def _scan_der_cert(der_certs):
current = -1
result = []
for log_index, der_cert, der_chain, entry_type in der_certs:
try:
current = log_index
certificate = None
strict_failure = False
try:
certificate = cert.Certificate(der_cert)
except error.Error as e:
try:
certificate = cert.Certificate(der_cert, strict_der=False)
except error.Error as e:
strict_failure = True
if not strict_failure:
desc = cert_desc.from_cert(certificate)
else:
desc = certificate_pb2.X509Description()
desc.der = der_cert
desc.sha256_hash = hashlib.sha256(der_cert).digest()
desc.entry_type = entry_type
root = None
if der_chain:
try:
issuer = cert.Certificate(der_chain[0], strict_der=False)
except error.Error:
pass
else:
desc.issuer_pk_sha256_hash = issuer.key_hash(
hashfunc="sha256")
try:
root = cert.Certificate(der_chain[-1], strict_der=False)
except error.Error:
pass
else:
# No chain implies this is a root certificate.
# Note that certificate may be None.
root = certificate
if root:
for iss in [
(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in root.issuer()
]:
proto_iss = desc.root_issuer.add()
proto_iss.type, proto_iss.value = iss
result.append((desc, log_index))
except:
batch_start_index, batch_end_index = (der_certs[0][0],
der_certs[-1][0])
logging.exception(
"Error scanning certificate %d in batch <%d, %d> - it will "
"be excluded from the scan results", current,
batch_start_index, batch_end_index)
return result
def _scan_der_cert(der_certs, checks):
current = -1
result = []
for log_index, der_cert, der_chain, entry_type in der_certs:
try:
current = log_index
partial_result = []
certificate = None
strict_failure = False
try:
certificate = cert.Certificate(der_cert)
except error.Error as e:
try:
certificate = cert.Certificate(der_cert, strict_der=False)
except error.Error as e:
partial_result.append(asn1.All())
strict_failure = True
else:
if isinstance(e, error.ASN1IllegalCharacter):
partial_result.append(asn1.Strict(reason=e.args[0],
details=(e.string, e.index)))
else:
partial_result.append(asn1.Strict(reason=str(e)))
if not strict_failure:
for check in checks:
partial_result += check.check(certificate) or []
desc = cert_desc.from_cert(certificate, partial_result)
else:
desc = certificate_pb2.X509Description()
desc.der = der_cert
desc.sha256_hash = hashlib.sha256(der_cert).digest()
desc.entry_type = entry_type
root = None
if der_chain:
try:
issuer = cert.Certificate(der_chain[0], strict_der=False)
except error.Error:
pass
else:
desc.issuer_pk_sha256_hash = issuer.key_hash(hashfunc="sha256")
try:
root = cert.Certificate(der_chain[-1], strict_der=False)
except error.Error:
pass
else:
# No chain implies this is a root certificate.
# Note that certificate may be None.
root = certificate
if root:
for iss in [(type_.short_name, cert_desc.to_unicode(
'.'.join(cert_desc.process_name(value.human_readable()))))
for type_, value in root.issuer()]:
proto_iss = desc.root_issuer.add()
proto_iss.type, proto_iss.value = iss
result.append((desc, log_index, partial_result))
except:
batch_start, batch_end = der_certs[0][0], der_certs[-1][0]
logging.exception(
"Error scanning certificate %d in batch <%d, %d> - it will "
"be excluded from the scan results",
current, batch_start, batch_end)
return result
def test_from_cert(self):
for test_case in [(CERT, False), (DSA_SHA256_CERT, False), (CA_CERT, True)]:
(source, expect_ca_true) = test_case
observations = []
for check in all_checks.ALL_CHECKS:
observations += check.check(source) or []
observations.append(observation.Observation(
"AE", u'ćę©ß→æ→ćąßę-ß©ąńśþa©ęńć←', (u'əꔹłęµ', u'…łą↓ð→↓ś→ę')))
proto = cert_desc.from_cert(source, observations)
self.assertEqual(proto.der, source.to_der())
subject = [(att.type, att.value) for att in proto.subject]
cert_subject = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in source.subject()]
self.assertItemsEqual(cert_subject, subject)
issuer = [(att.type, att.value) for att in proto.issuer]
cert_issuer = [(type_.short_name,
cert_desc.to_unicode('.'.join(
cert_desc.process_name(value.human_readable()))))
for type_, value in source.issuer()]
self.assertItemsEqual(cert_issuer, issuer)
subject_alternative_names = [(att.type, att.value)
for att in proto.subject_alternative_names]
cert_subject_alternative_names = [(san.component_key(),
cert_desc.to_unicode('.'.join(
cert_desc.process_name(
san.component_value().human_readable()))))
for san in source.subject_alternative_names()]
self.assertItemsEqual(cert_subject_alternative_names,
subject_alternative_names)
self.assertEqual(proto.version, str(source.version().value))
self.assertEqual(proto.serial_number,
str(source.serial_number().human_readable()
.upper().replace(':', '')))
self.assertEqual(time.gmtime(proto.validity.not_before / 1000),
source.not_before())
self.assertEqual(time.gmtime(proto.validity.not_after / 1000),
source.not_after())
observations_tuples = [(unicode(obs.description),
unicode(obs.reason) if obs.reason else u'',
obs.details_to_proto())
for obs in observations]
proto_obs = [(obs.description, obs.reason, obs.details)
for obs in proto.observations]
self.assertItemsEqual(proto_obs, observations_tuples)
self.assertEqual(proto.tbs_signature.algorithm_id,
source.signature()["algorithm"].long_name)
self.assertEqual(proto.cert_signature.algorithm_id,
source.signature_algorithm()["algorithm"].long_name)
self.assertEqual(proto.tbs_signature.algorithm_id,
proto.cert_signature.algorithm_id)
if source.signature()["parameters"]:
self.assertEqual(proto.tbs_signature.parameters,
source.signature()["parameters"])
else:
self.assertFalse(proto.tbs_signature.HasField('parameters'))
if source.signature_algorithm()["parameters"]:
self.assertEqual(proto.cert_signature.parameters,
source.signature_algorithm()["parameters"])
else:
self.assertFalse(proto.cert_signature.HasField('parameters'))
self.assertEqual(proto.tbs_signature.parameters,
proto.cert_signature.parameters)
self.assertEqual(proto.basic_constraint_ca, expect_ca_true)