def add_new_certificate_values(new_ids, ct_collection, zones, save_location=None): """ Add new certificate values to the database. """ x509_parser = X509Parser.X509Parser() existing_ids = get_list_of_existing_certificates(ct_collection) for min_cert_id in new_ids: if min_cert_id not in existing_ids: # Pace out certificate requests against their service time.sleep(2) c_file = make_https_request("https://crt.sh/?d=" + str(min_cert_id), True) if c_file is None: print("ERROR: Failed communicating with crt.sh. Skipping cert_id: " + str(min_cert_id)) continue if save_location is not None: open(save_location + str(min_cert_id) + ".crt", "wb").write(c_file) cert = x509_parser.parse_data(c_file, "crt_sh") if cert is None: print("ERROR: Could not parse certificate for: " + str(min_cert_id) + ". Skipping for now.") continue cert_zones = get_cert_zones(cert, zones) print("Adding crt.sh id: " + str(min_cert_id) + " SHA256: " + cert['fingerprint_sha256']) if ct_collection.find({"fingerprint_sha256": cert['fingerprint_sha256']}).count() != 0: # The certificate exists in the database but does not have crt_sh id and/or zones ct_collection.update_one({"fingerprint_sha256": cert['fingerprint_sha256']}, {"$set": {"crt_sh_min_id": min_cert_id, "zones": cert_zones}, "$addToSet": {'sources': 'crt_sh'}}) else: # Add the new certificate cert['crt_sh_min_id'] = min_cert_id cert['zones'] = cert_zones ct_collection.insert_one(cert)
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Make database connections mongo_connector = MongoConnector.MongoConnector() ct_collection = mongo_connector.get_certificate_transparency_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, "facebook_certs") jobs_manager.record_job_start() file_path = "/mnt/workspace/ct_facebook/" fb_connector = FacebookConnector.FacebookConnector() access_token = fb_connector.get_facebook_access_token() zones = ZoneManager.get_distinct_zones(mongo_connector) x509_parser = X509Parser.X509Parser() parser = argparse.ArgumentParser( description='Download DNS and/or certificate information from crt.sh.') parser.add_argument( '--fetch_cert_records', choices=['dbAndSave', 'dbOnly'], default="dbAndSave", help= 'Indicates whether to download the raw files or just record in the database' ) parser.add_argument( '--cert_save_location', required=False, default=file_path, help= 'Indicates where to save the certificates on disk when choosing dbAndSave' ) args = parser.parse_args() check_save_location(args.cert_save_location) save_location = args.cert_save_location if not save_location.endswith("/"): save_location = save_location + "/" for zone in zones: time.sleep(15) results = fetch_domain(logger, fb_connector, access_token, zone) if results is None: logger.warning("ERROR looking up: " + zone) continue logger.info(zone + ": " + str(len(results))) for result in results: if args.fetch_cert_records == "dbAndSave": cert_f = open( save_location + zone + "_" + result['id'] + ".pem", "w") cert_f.write(result['certificate_pem']) cert_f.close() cert = x509_parser.parse_data(result['certificate_pem'], "facebook") cert['facebook_id'] = result['id'] if ct_collection.find({ 'fingerprint_sha256': cert['fingerprint_sha256'] }).count() == 0: ct_collection.insert(cert) else: if ct_collection.find({ 'fingerprint_sha256': cert['fingerprint_sha256'], 'facebook_id': result['id'], 'zones': zone }).count() == 0: ct_collection.update( {'fingerprint_sha256': cert['fingerprint_sha256']}, { "$set": { 'marinus_updated': datetime.now(), 'facebook_id': result['id'] }, "$addToSet": { 'zones': zone } }) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Make database connections mongo_connector = MongoConnector.MongoConnector() ct_collection = mongo_connector.get_certificate_transparency_connection() config_collection = mongo_connector.get_config_connection() x509parser = X509Parser.X509Parser() zones = ZoneManager.get_distinct_zones(mongo_connector) result = config_collection.find_one({}, {'SSL_Orgs': 1, "_id": 0}) ssl_orgs = result['SSL_Orgs'] # Defaults save_location = '/mnt/workspace/' download_method = 'dbAndSave' save_type = "PEM" parser = argparse.ArgumentParser( description='Download certificate information from the provide CT Log.' ) parser.add_argument( '--log_source', required=True, help= 'Indicates which log to query based on values in the x509Parser library' ) parser.add_argument( '--include_precerts', action="store_true", help='Include pre-certificates which are not finalized') parser.add_argument( '--download_methods', choices=['dbAndSave', 'dbOnly'], default=download_method, help= 'Indicates whether to download the raw files or just save to the database' ) parser.add_argument( '--starting_index', required=False, default=-1, type=int, help='Force the script to start at specific index within the log.') parser.add_argument( '--cert_save_location', required=False, default=save_location, help= 'Indicates where to save the certificates on disk when choosing dbAndSave' ) parser.add_argument( '--save_type', choices=['PEM', 'ASN1'], default=save_type, help='Indicates which format to use for the data. The default is PEM') args = parser.parse_args() source = args.log_source try: ct_log_map = x509parser.CT_LOG_MAP[source] except: logger.error("ERROR: UNKNOWN LOG SOURCE: " + source) exit(1) if args.cert_save_location: save_location = args.cert_save_location if not save_location.endswith("/"): save_location = save_location + "/" if args.download_methods: download_method = args.download_methods check_save_location(save_location, source) if args.save_type: save_type = args.save_type jobs_manager = JobsManager.JobsManager(mongo_connector, "ct_log-" + source) jobs_manager.record_job_start() if args.starting_index == -1: starting_index = fetch_starting_index(ct_collection, source) else: starting_index = args.starting_index logger.info("Starting Index: " + str(starting_index)) sth_data = fetch_sth(logger, "https://" + ct_log_map['url']) logger.info("Tree size: " + str(sth_data['tree_size'])) current_index = starting_index while current_index < sth_data['tree_size']: ending_index = current_index + 256 if ending_index > sth_data['tree_size']: ending_index = sth_data['tree_size'] logger.debug("Checking from index: " + str(current_index) + " to index " + str(ending_index)) certs = fetch_certificate_batch(logger, "https://" + ct_log_map['url'], current_index, ending_index) for entry in certs['entries']: der_cert, cert_type = get_cert_from_leaf(logger, entry['leaf_input']) if der_cert is None and cert_type == 1 and not args.include_precerts: current_index = current_index + 1 continue elif der_cert is None and cert_type == 0: current_index = current_index + 1 continue elif der_cert is None and cert_type == 1: der_cert = get_cert_from_extra_data(entry['extra_data']) cert = x509parser.parse_data(der_cert, source) if cert is None: logger.warning("Skipping certificate index: " + str(current_index)) current_index = current_index + 1 continue if cert_type == 1: cert['ct_log_type'] = "PRE-CERTIFICATE" else: cert['ct_log_type'] = "CERTIFICATE" cert_zones = check_zone_relevancy(cert, zones) if check_org_relevancy(cert, ssl_orgs) or cert_zones != []: cert[source + "_id"] = current_index cert['zones'] = cert_zones logger.info("Adding " + source + " id: " + str(current_index) + " SHA256: " + cert['fingerprint_sha256']) insert_certificate(cert, source, ct_collection, cert_zones) if download_method == 'dbAndSave': write_file(logger, cert, save_location, save_type, source) current_index = current_index + 1 # Set isExpired for any entries that have recently expired. ct_collection.update( { "not_after": { "$lt": datetime.utcnow() }, "isExpired": False }, {"$set": { "isExpired": True }}, multi=True) jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def add_new_certificate_values( logger, jobs_manager, new_ids, ct_collection, zones, save_location=None ): """ Add new certificate values to the database. """ x509_parser = X509Parser.X509Parser() existing_ids = get_list_of_existing_certificates(ct_collection) for min_cert_id in new_ids: if min_cert_id not in existing_ids: # Pace out certificate requests against their service time.sleep(2) c_file = make_https_request( logger, "https://crt.sh/?d=" + str(min_cert_id), jobs_manager, True ) if c_file is None: logger.warning( "ERROR: Failed communicating with crt.sh. Skipping cert_id: " + str(min_cert_id) ) continue if save_location is not None: open(save_location + str(min_cert_id) + ".crt", "wb").write(c_file) cert = x509_parser.parse_data(c_file, "crt_sh") if cert is None: logger.warning( "ERROR: Could not parse certificate for: " + str(min_cert_id) + ". Skipping for now." ) continue cert_zones = get_cert_zones(cert, zones) logger.info( "Adding crt.sh id: " + str(min_cert_id) + " SHA256: " + cert["fingerprint_sha256"] ) if ( ct_collection.count_documents( {"fingerprint_sha256": cert["fingerprint_sha256"]} ) != 0 ): # The certificate exists in the database but does not have crt_sh id and/or zones ct_collection.update_one( {"fingerprint_sha256": cert["fingerprint_sha256"]}, { "$set": { "crt_sh_min_id": min_cert_id, "zones": cert_zones, "marinus_updated": datetime.now(), }, "$addToSet": {"sources": "crt_sh"}, }, ) else: # Add the new certificate cert["crt_sh_min_id"] = min_cert_id cert["zones"] = cert_zones ct_collection.insert_one(cert)