def add_new_certificate_values(new_ids, ct_collection, zones, save_location=None):
"""
Add new certificate values to the database.
"""
x509_parser = X509Parser.X509Parser()
existing_ids = get_list_of_existing_certificates(ct_collection)
for min_cert_id in new_ids:
if min_cert_id not in existing_ids:
# Pace out certificate requests against their service
time.sleep(2)
c_file = make_https_request("https://crt.sh/?d=" + str(min_cert_id), True)
if c_file is None:
print("ERROR: Failed communicating with crt.sh. Skipping cert_id: " + str(min_cert_id))
continue
if save_location is not None:
open(save_location + str(min_cert_id) + ".crt", "wb").write(c_file)
cert = x509_parser.parse_data(c_file, "crt_sh")
if cert is None:
print("ERROR: Could not parse certificate for: " + str(min_cert_id) + ". Skipping for now.")
continue
cert_zones = get_cert_zones(cert, zones)
print("Adding crt.sh id: " + str(min_cert_id) + " SHA256: " + cert['fingerprint_sha256'])
if ct_collection.find({"fingerprint_sha256": cert['fingerprint_sha256']}).count() != 0:
# The certificate exists in the database but does not have crt_sh id and/or zones
ct_collection.update_one({"fingerprint_sha256": cert['fingerprint_sha256']}, {"$set": {"crt_sh_min_id": min_cert_id, "zones": cert_zones}, "$addToSet": {'sources': 'crt_sh'}})
else:
# Add the new certificate
cert['crt_sh_min_id'] = min_cert_id
cert['zones'] = cert_zones
ct_collection.insert_one(cert)
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Make database connections
mongo_connector = MongoConnector.MongoConnector()
ct_collection = mongo_connector.get_certificate_transparency_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector, "facebook_certs")
jobs_manager.record_job_start()
file_path = "/mnt/workspace/ct_facebook/"
fb_connector = FacebookConnector.FacebookConnector()
access_token = fb_connector.get_facebook_access_token()
zones = ZoneManager.get_distinct_zones(mongo_connector)
x509_parser = X509Parser.X509Parser()
parser = argparse.ArgumentParser(
description='Download DNS and/or certificate information from crt.sh.')
parser.add_argument(
'--fetch_cert_records',
choices=['dbAndSave', 'dbOnly'],
default="dbAndSave",
help=
'Indicates whether to download the raw files or just record in the database'
)
parser.add_argument(
'--cert_save_location',
required=False,
default=file_path,
help=
'Indicates where to save the certificates on disk when choosing dbAndSave'
)
args = parser.parse_args()
check_save_location(args.cert_save_location)
save_location = args.cert_save_location
if not save_location.endswith("/"):
save_location = save_location + "/"
for zone in zones:
time.sleep(15)
results = fetch_domain(logger, fb_connector, access_token, zone)
if results is None:
logger.warning("ERROR looking up: " + zone)
continue
logger.info(zone + ": " + str(len(results)))
for result in results:
if args.fetch_cert_records == "dbAndSave":
cert_f = open(
save_location + zone + "_" + result['id'] + ".pem", "w")
cert_f.write(result['certificate_pem'])
cert_f.close()
cert = x509_parser.parse_data(result['certificate_pem'],
"facebook")
cert['facebook_id'] = result['id']
if ct_collection.find({
'fingerprint_sha256':
cert['fingerprint_sha256']
}).count() == 0:
ct_collection.insert(cert)
else:
if ct_collection.find({
'fingerprint_sha256':
cert['fingerprint_sha256'],
'facebook_id':
result['id'],
'zones':
zone
}).count() == 0:
ct_collection.update(
{'fingerprint_sha256': cert['fingerprint_sha256']}, {
"$set": {
'marinus_updated': datetime.now(),
'facebook_id': result['id']
},
"$addToSet": {
'zones': zone
}
})
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Make database connections
mongo_connector = MongoConnector.MongoConnector()
ct_collection = mongo_connector.get_certificate_transparency_connection()
config_collection = mongo_connector.get_config_connection()
x509parser = X509Parser.X509Parser()
zones = ZoneManager.get_distinct_zones(mongo_connector)
result = config_collection.find_one({}, {'SSL_Orgs': 1, "_id": 0})
ssl_orgs = result['SSL_Orgs']
# Defaults
save_location = '/mnt/workspace/'
download_method = 'dbAndSave'
save_type = "PEM"
parser = argparse.ArgumentParser(
description='Download certificate information from the provide CT Log.'
)
parser.add_argument(
'--log_source',
required=True,
help=
'Indicates which log to query based on values in the x509Parser library'
)
parser.add_argument(
'--include_precerts',
action="store_true",
help='Include pre-certificates which are not finalized')
parser.add_argument(
'--download_methods',
choices=['dbAndSave', 'dbOnly'],
default=download_method,
help=
'Indicates whether to download the raw files or just save to the database'
)
parser.add_argument(
'--starting_index',
required=False,
default=-1,
type=int,
help='Force the script to start at specific index within the log.')
parser.add_argument(
'--cert_save_location',
required=False,
default=save_location,
help=
'Indicates where to save the certificates on disk when choosing dbAndSave'
)
parser.add_argument(
'--save_type',
choices=['PEM', 'ASN1'],
default=save_type,
help='Indicates which format to use for the data. The default is PEM')
args = parser.parse_args()
source = args.log_source
try:
ct_log_map = x509parser.CT_LOG_MAP[source]
except:
logger.error("ERROR: UNKNOWN LOG SOURCE: " + source)
exit(1)
if args.cert_save_location:
save_location = args.cert_save_location
if not save_location.endswith("/"):
save_location = save_location + "/"
if args.download_methods:
download_method = args.download_methods
check_save_location(save_location, source)
if args.save_type:
save_type = args.save_type
jobs_manager = JobsManager.JobsManager(mongo_connector, "ct_log-" + source)
jobs_manager.record_job_start()
if args.starting_index == -1:
starting_index = fetch_starting_index(ct_collection, source)
else:
starting_index = args.starting_index
logger.info("Starting Index: " + str(starting_index))
sth_data = fetch_sth(logger, "https://" + ct_log_map['url'])
logger.info("Tree size: " + str(sth_data['tree_size']))
current_index = starting_index
while current_index < sth_data['tree_size']:
ending_index = current_index + 256
if ending_index > sth_data['tree_size']:
ending_index = sth_data['tree_size']
logger.debug("Checking from index: " + str(current_index) +
" to index " + str(ending_index))
certs = fetch_certificate_batch(logger, "https://" + ct_log_map['url'],
current_index, ending_index)
for entry in certs['entries']:
der_cert, cert_type = get_cert_from_leaf(logger,
entry['leaf_input'])
if der_cert is None and cert_type == 1 and not args.include_precerts:
current_index = current_index + 1
continue
elif der_cert is None and cert_type == 0:
current_index = current_index + 1
continue
elif der_cert is None and cert_type == 1:
der_cert = get_cert_from_extra_data(entry['extra_data'])
cert = x509parser.parse_data(der_cert, source)
if cert is None:
logger.warning("Skipping certificate index: " +
str(current_index))
current_index = current_index + 1
continue
if cert_type == 1:
cert['ct_log_type'] = "PRE-CERTIFICATE"
else:
cert['ct_log_type'] = "CERTIFICATE"
cert_zones = check_zone_relevancy(cert, zones)
if check_org_relevancy(cert, ssl_orgs) or cert_zones != []:
cert[source + "_id"] = current_index
cert['zones'] = cert_zones
logger.info("Adding " + source + " id: " + str(current_index) +
" SHA256: " + cert['fingerprint_sha256'])
insert_certificate(cert, source, ct_collection, cert_zones)
if download_method == 'dbAndSave':
write_file(logger, cert, save_location, save_type, source)
current_index = current_index + 1
# Set isExpired for any entries that have recently expired.
ct_collection.update(
{
"not_after": {
"$lt": datetime.utcnow()
},
"isExpired": False
}, {"$set": {
"isExpired": True
}},
multi=True)
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def add_new_certificate_values(
logger, jobs_manager, new_ids, ct_collection, zones, save_location=None
):
"""
Add new certificate values to the database.
"""
x509_parser = X509Parser.X509Parser()
existing_ids = get_list_of_existing_certificates(ct_collection)
for min_cert_id in new_ids:
if min_cert_id not in existing_ids:
# Pace out certificate requests against their service
time.sleep(2)
c_file = make_https_request(
logger, "https://crt.sh/?d=" + str(min_cert_id), jobs_manager, True
)
if c_file is None:
logger.warning(
"ERROR: Failed communicating with crt.sh. Skipping cert_id: "
+ str(min_cert_id)
)
continue
if save_location is not None:
open(save_location + str(min_cert_id) + ".crt", "wb").write(c_file)
cert = x509_parser.parse_data(c_file, "crt_sh")
if cert is None:
logger.warning(
"ERROR: Could not parse certificate for: "
+ str(min_cert_id)
+ ". Skipping for now."
)
continue
cert_zones = get_cert_zones(cert, zones)
logger.info(
"Adding crt.sh id: "
+ str(min_cert_id)
+ " SHA256: "
+ cert["fingerprint_sha256"]
)
if (
ct_collection.count_documents(
{"fingerprint_sha256": cert["fingerprint_sha256"]}
)
!= 0
):
# The certificate exists in the database but does not have crt_sh id and/or zones
ct_collection.update_one(
{"fingerprint_sha256": cert["fingerprint_sha256"]},
{
"$set": {
"crt_sh_min_id": min_cert_id,
"zones": cert_zones,
"marinus_updated": datetime.now(),
},
"$addToSet": {"sources": "crt_sh"},
},
)
else:
# Add the new certificate
cert["crt_sh_min_id"] = min_cert_id
cert["zones"] = cert_zones
ct_collection.insert_one(cert)