def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Make database connections mongo_connector = MongoConnector.MongoConnector() ct_collection = mongo_connector.get_certificate_transparency_connection() config_collection = mongo_connector.get_config_connection() x509parser = X509Parser.X509Parser() zones = ZoneManager.get_distinct_zones(mongo_connector) result = config_collection.find_one({}, {"SSL_Orgs": 1, "_id": 0}) ssl_orgs = result["SSL_Orgs"] # Defaults save_location = "/mnt/workspace/" download_method = "dbAndSave" save_type = "PEM" parser = argparse.ArgumentParser( description="Download certificate information from the provide CT Log." ) parser.add_argument( "--log_source", required=True, help= "Indicates which log to query based on values in the x509Parser library", ) parser.add_argument( "--include_precerts", action="store_true", help="Include pre-certificates which are not finalized", ) parser.add_argument( "--download_methods", choices=["dbAndSave", "dbOnly"], default=download_method, help= "Indicates whether to download the raw files or just save to the database", ) parser.add_argument( "--starting_index", required=False, default=-1, type=int, help="Force the script to start at specific index within the log.", ) parser.add_argument( "--cert_save_location", required=False, default=save_location, help= "Indicates where to save the certificates on disk when choosing dbAndSave", ) parser.add_argument( "--save_type", choices=["PEM", "ASN1"], default=save_type, help="Indicates which format to use for the data. The default is PEM", ) args = parser.parse_args() source = args.log_source try: ct_log_map = x509parser.CT_LOG_MAP[source] except: logger.error("ERROR: UNKNOWN LOG SOURCE: " + source) exit(1) if args.cert_save_location: save_location = args.cert_save_location if not save_location.endswith("/"): save_location = save_location + "/" if args.download_methods: download_method = args.download_methods check_save_location(save_location, source) if args.save_type: save_type = args.save_type jobs_manager = JobsManager.JobsManager(mongo_connector, "ct_log-" + source) jobs_manager.record_job_start() if args.starting_index == -1: starting_index = fetch_starting_index(ct_collection, source) else: starting_index = args.starting_index logger.info("Starting Index: " + str(starting_index)) sth_data = fetch_sth(logger, "https://" + ct_log_map["url"], jobs_manager) logger.info("Tree size: " + str(sth_data["tree_size"])) current_index = starting_index while current_index < sth_data["tree_size"]: ending_index = current_index + 256 if ending_index > sth_data["tree_size"]: ending_index = sth_data["tree_size"] logger.debug("Checking from index: " + str(current_index) + " to index " + str(ending_index)) certs = fetch_certificate_batch( logger, "https://" + ct_log_map["url"], current_index, ending_index, jobs_manager, ) for entry in certs["entries"]: der_cert, cert_type = get_cert_from_leaf(logger, entry["leaf_input"]) if der_cert is None and cert_type == 1 and not args.include_precerts: current_index = current_index + 1 continue elif der_cert is None and cert_type == 0: current_index = current_index + 1 continue elif der_cert is None and cert_type == 1: der_cert = get_cert_from_extra_data(entry["extra_data"]) cert = x509parser.parse_data(der_cert, source) if cert is None: logger.warning("Skipping certificate index: " + str(current_index)) current_index = current_index + 1 continue if cert_type == 1: cert["ct_log_type"] = "PRE-CERTIFICATE" else: cert["ct_log_type"] = "CERTIFICATE" cert_zones = check_zone_relevancy(cert, zones) if check_org_relevancy(cert, ssl_orgs) or cert_zones != []: cert[source + "_id"] = current_index cert["zones"] = cert_zones logger.info("Adding " + source + " id: " + str(current_index) + " SHA256: " + cert["fingerprint_sha256"]) insert_certificate(cert, source, ct_collection, cert_zones) if download_method == "dbAndSave": write_file(logger, cert, save_location, save_type, source) current_index = current_index + 1 # Set isExpired for any entries that have recently expired. ct_collection.update_many( { "not_after": { "$lt": datetime.utcnow() }, "isExpired": False }, {"$set": { "isExpired": True }}, ) jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) # Make database connections mongo_connector = MongoConnector.MongoConnector() ct_collection = mongo_connector.get_certificate_transparency_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, "facebook_certs") jobs_manager.record_job_start() file_path = "/mnt/workspace/ct_facebook/" fb_connector = FacebookConnector.FacebookConnector() access_token = fb_connector.get_facebook_access_token() zones = ZoneManager.get_distinct_zones(mongo_connector) x509_parser = X509Parser.X509Parser() parser = argparse.ArgumentParser( description='Download DNS and/or certificate information from crt.sh.') parser.add_argument( '--fetch_cert_records', choices=['dbAndSave', 'dbOnly'], default="dbAndSave", help= 'Indicates whether to download the raw files or just record in the database' ) parser.add_argument( '--cert_save_location', required=False, default=file_path, help= 'Indicates where to save the certificates on disk when choosing dbAndSave' ) args = parser.parse_args() check_save_location(args.cert_save_location) save_location = args.cert_save_location if not save_location.endswith("/"): save_location = save_location + "/" for zone in zones: time.sleep(15) results = fetch_domain(fb_connector, access_token, zone) if results is None: print("ERROR looking up: " + zone) continue print(zone + ": " + str(len(results))) for result in results: if args.fetch_cert_records == "dbAndSave": cert_f = open( save_location + zone + "_" + result['id'] + ".pem", "w") cert_f.write(result['certificate_pem']) cert_f.close() cert = x509_parser.parse_data(result['certificate_pem'], "facebook") cert['facebook_id'] = result['id'] if ct_collection.find({ 'fingerprint_sha256': cert['fingerprint_sha256'] }).count() == 0: ct_collection.insert(cert) else: ct_collection.update( {'fingerprint_sha256': cert['fingerprint_sha256']}, {"$addToSet": { 'zones': zone }}) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now))
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) google_dns = GoogleDNS.GoogleDNS() jobs_manager = JobsManager.JobsManager(mongo_connector, 'extract_ssl_domains') jobs_manager.record_job_start() parser = argparse.ArgumentParser( description='Search TLS certificates for additional DNS names') parser.add_argument('--zgrab_version', default=2, type=int, choices=[1, 2], metavar="version", help='The version of ZGrab used to collect data') args = parser.parse_args() dns_names = [] round_two = [] zones = ZoneManager.get_distinct_zones(mongo_connector) # Collect the list of domains from the SSL Certificates extract_ct_certificate_names(dns_names, mongo_connector) # extract_censys_certificate_names(dns_names, mongo_connector) if args.zgrab_version == 1: extract_zgrab_certificate_names(dns_names, mongo_connector) else: extract_zgrab2_certificate_names(dns_names, mongo_connector) input_list = [] # Some SSL certificates are for multiple domains. # The tracked company may not own all domains. # Therefore, we filter to only the root domains that belong to the tracked company. print("Pre-filter list: " + str(len(dns_names))) for hostname in dns_names: if not hostname.startswith("*"): zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) # Pause to prevent DoS-ing of Google's HTTPS DNS Service time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) if ip_addr['type'] == "cname" and is_tracked_zone( ip_addr['value'], zones): add_to_round_two(ip_addr['value'], round_two) else: print("Failed IP Lookup for: " + hostname) else: print("Failed match on zone for: " + hostname) else: print("Skipping wildcard: " + hostname) dead_dns_collection = mongo_connector.get_dead_dns_connection() # Some DNS records will be CNAME records pointing to other tracked domains. # This is a single level recursion to lookup those domains. print("Round Two list: " + str(len(round_two))) for hostname in round_two: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) else: print("Failed IP Lookup for: " + hostname) original_record = dns_manager.find_one({"fqdn": hostname}, "ssl") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) else: print("Failed match on zone for: " + hostname) # Record all the results. dns_manager.remove_by_source("ssl") print("List length: " + str(len(input_list))) for final_result in input_list: dns_manager.insert_record(final_result, "ssl") # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now))
def main(): """ Begin Main """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Set up all the database connections mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'create_netaddr_graphs') jobs_manager.record_job_start() # Get the list of the all Class C's in Marinus groups = [] create_list_of_cidrs(groups, mongo_connector, dns_manager) # Create a separate copy of the class C list since groups will be modified later cidr_list = groups + [] # Create the stats on the network data group_data = create_network_data_sets(groups, mongo_connector) logger.info("Number of Tracked Class C's: " + str(group_data['tracked_count'])) logger.info("Number of AWS Class C's: " + str(group_data['aws_count'])) logger.info("Number of Azure Class C's: " + str(group_data['azure_count'])) logger.info("Number of Akamai Class C's: " + str(group_data['akamai_count'])) logger.info("Number of Class C's: " + str(len(groups))) # Get the current list of zones zones = ZoneManager.get_distinct_zones(mongo_connector) # For each Class C that was identified in Marinus... for tcidr in cidr_list: cidr = tcidr.replace(REPLACE_CHAR, ".") groups = [] graph = nx.Graph() add_to_list(cidr, groups) graph.add_node(cidr, data_type="class_c", type=0, depends=[], dependedOnBy=[], docs="<h1>Parent</h1>") find_all_dns_by_zone(graph, cidr, groups, dns_manager) find_srdns_by_zone(graph, cidr, groups, mongo_connector) data = json_graph.node_link_data(graph) reformat_data(data, cidr, groups) new_data = {} new_data['directed'] = data['directed'] new_data['graph'] = data['graph'] new_data['multigraph'] = data['multigraph'] new_data['errs'] = [] new_data['links'] = data['links'] new_data['data'] = {} for i in range(0, len(data['nodes'])): new_data['data'][data['nodes'][i]['id'].replace( ".", REPLACE_CHAR)] = data['nodes'][i] config = {} config['title'] = cidr + " Network Map" config['graph'] = {} config['graph']['linkDistance'] = 150 config['graph']['charge'] = -400 config['graph']['height'] = 800 config['graph']['numColors'] = len(groups) config['graph']['labelPadding'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['labelMargin'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['ticksWithoutCollisions'] = 50 config['graph_type'] = "cidr" config['types'] = {} regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$" regx = re.compile(regex_str) for tgroup in groups: group = tgroup.replace(REPLACE_CHAR, ".") data_type = "tpd" if group in zones: data_type = "tracked_domain" elif re.match(regx, group): data_type = "cidr" config['types'][tgroup] = { "short": group, "long": "A group from the network: " + group, "data_type": data_type } config['constraints'] = [] tmp = int(math.ceil(math.sqrt(len(groups)))) + 1 x = [] y = [] for i in range(1, tmp): val = round((i * 1.0) / tmp, 2) x.append(str(val)) y.append(str(val)) x_pos = 0 y_pos = 0 for group in groups: config['constraints'].append({ "has": { "type": group }, "type": "position", "x": x[x_pos], "y": y[y_pos] }) x_pos = x_pos + 1 if x_pos >= len(x): x_pos = 0 y_pos = y_pos + 1 config['jsonUrl'] = "/api/v1.0/cidr_graphs/" + cidr new_data['config'] = config new_data['created'] = datetime.now() new_data['zone'] = cidr cidr_graphs_collection = mongo_connector.get_cidr_graphs_connection() cidr_graphs_collection.remove({'zone': cidr}) cidr_graphs_collection.insert_one(new_data) time.sleep(1) # Remove last week's old entries lastweek = datetime.now() - timedelta(days=7) cidr_graphs_collection.remove({'created': {"$lt": lastweek}}) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): global global_exit_flag global global_zgrab_path parser = argparse.ArgumentParser( description='Launch zgrab against IPs using port 80 or 443.') parser.add_argument('-p', choices=['443', '80'], metavar="port", help='The web port: 80 or 443') parser.add_argument('-t', default=5, type=int, metavar="threadCount", help='The number of threads') parser.add_argument('--zgrab_path', default=global_zgrab_path, metavar='zgrabVersion', help='The version of ZGrab to use') args = parser.parse_args() if args.p == None: print("A port value (80 or 443) must be provided.") exit(0) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() print(str(now) + ": I am already running! Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) rm_connector = RemoteMongoConnector.RemoteMongoConnector() all_dns_collection = rm_connector.get_all_dns_connection() ip_manager = IPManager.IPManager(rm_connector, True) jobs_manager = JobsManager.JobsManager(rm_connector, "zgrab_http_ip-" + args.p) jobs_manager.record_job_start() zones_struct = {} zones_struct['zones'] = ZoneManager.get_distinct_zones(rm_connector) # Not pretty but cleaner than previous method zones_struct['ip_manager'] = ip_manager (ips, ip_context) = get_ips(ip_manager, all_dns_collection) print("Got IPs: " + str(len(ips))) zones_struct['ip_context'] = ip_context if args.p == "443": zgrab_collection = rm_connector.get_zgrab_443_data_connection() run_command = run_port_443_command else: zgrab_collection = rm_connector.get_zgrab_80_data_connection() run_command = run_port_80_command check_save_location("./json_p" + args.p) global_zgrab_path = args.zgrab_path threads = [] print("Creating " + str(args.t) + " threads") for thread_id in range(1, args.t + 1): thread = ZgrabThread(thread_id, global_work_queue, args.p, run_command, zones_struct, zgrab_collection) thread.start() threads.append(thread) thread_id += 1 print("Populating Queue") global_queue_lock.acquire() for ip in ips: global_work_queue.put(ip) global_queue_lock.release() # Wait for queue to empty while not global_work_queue.empty(): pass # Notify threads it's time to exit global_exit_flag = 1 # Wait for all threads to complete for t in threads: t.join() print("Exiting Main Thread") # Remove last week's old entries lastweek = datetime.now() - timedelta(days=7) zgrab_collection.remove({ 'ip': { "$ne": "<nil>" }, 'timestamp': { "$lt": lastweek } }) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now))
def main(): """ Begin Main... """ # The sources for which to remove expired entries # Infoblox is handled separately # Sonar RDNS is hard code below in a separate section # {"source_name": date_difference_in_months} sources = [{"name": "sonar_dns", "diff": -2}, {"name": "sonar_dns_saved", "diff": -2}, {"name": "ssl", "diff": -2}, {"name": "ssl_saved", "diff": -2}, {"name": "virustotal", "diff": -2}, {"name": "virustotal_saved", "diff": -2}, {"name": "UltraDNS", "diff": -2}, {"name": "UltraDNS_saved", "diff": -2}, {"name": "skms", "diff": -2}, {"name": "skms_saved", "diff": -2}, {"name": "marinus", "diff": -2}, {"name": "marinus_saved", "diff": -2}, {"name": "mx", "diff": -2}, {"name": "mx_saved", "diff": -2}, {"name": "common_crawl", "diff": -4}, {"name": "common_crawl_saved", "diff": -4}] now = datetime.now() print ("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() all_dns_collection = mongo_connector.get_all_dns_connection() dns_manager = DNSManager.DNSManager(mongo_connector) GDNS = GoogleDNS.GoogleDNS() zones = ZoneManager.get_distinct_zones(mongo_connector) jobs_collection = mongo_connector.get_jobs_connection() # Get the date for today minus two months d_minus_2m = monthdelta(datetime.now(), -2) print("Removing SRDNS as of: " + str(d_minus_2m)) # Remove the old records srdns_collection = mongo_connector.get_sonar_reverse_dns_connection() srdns_collection.remove({'updated': {"$lt": d_minus_2m}}) # Before completely removing old entries, make an attempt to see if they are still valid. # Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source. # Rather than throw away valid information, this will archive it. for entry in sources: removal_date = monthdelta(datetime.now(), entry['diff']) source = entry['name'] print("Removing " + source + " as of: " + str(removal_date)) last_domain = "" results = all_dns_collection.find({'sources': {"$size": 1}, 'sources.source': source, 'sources.updated': {"$lt": removal_date}}) for result in results: if result['fqdn'] != last_domain: last_domain = result['fqdn'] dns_result = GDNS.fetch_DNS_records(result['fqdn'], GDNS.DNS_TYPES[result['type']]) if dns_result != []: for dns_entry in dns_result: if is_tracked_zone(dns_entry['fqdn'], zones): new_entry={} new_entry['updated'] = datetime.now() new_entry['zone'] = result['zone'] new_entry['fqdn'] = dns_entry['fqdn'] new_entry['created'] = result['created'] new_entry['value'] = dns_entry['value'] new_entry['type'] = dns_entry['type'] new_entry['status'] = 'confirmed' if 'sonar_timestamp' in result: new_entry['sonar_timestamp'] = result['sonar_timestamp'] if source.endswith("_saved"): dns_manager.insert_record(new_entry, source) else: dns_manager.insert_record(new_entry, source + "_saved") dns_manager.remove_all_by_source_and_date(source, entry['diff']) # Record status jobs_collection.update_one({'job_name': 'remove_expired_entries'}, {'$currentDate': {"updated": True}, "$set": {'status': 'COMPLETE'}}) now = datetime.now() print("Complete: " + str(now))
def main(): """ Begin Main """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) graphs_collection = mongo_connector.get_graphs_connection() graphs_data_collection = mongo_connector.get_graphs_data_connection() graphs_links_collection = mongo_connector.get_graphs_links_connection() graphs_docs_collection = mongo_connector.get_graphs_docs_connection() ip_manager = IPManager.IPManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'create_graphs2') jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) for zone in zones: groups = [] graph = nx.Graph() add_to_list(zone, groups) graph.add_node(zone, data_type="tld", type=0, depends=[], dependedOnBy=[], docs="<h1>Parent</h1>") find_all_dns_by_zone(graph, zone, groups, dns_manager, ip_manager) find_srdns_by_zone(graph, zone, groups, mongo_connector, ip_manager) data = json_graph.node_link_data(graph) reformat_data(data, zone, groups) new_data = {} new_data['directed'] = data['directed'] new_data['graph'] = data['graph'] new_data['multigraph'] = data['multigraph'] new_data['errs'] = [] config = {} config['title'] = zone + " Network Map" config['graph'] = {} config['graph']['linkDistance'] = 150 config['graph']['charge'] = -400 config['graph']['height'] = 800 config['graph']['numColors'] = len(groups) config['graph']['labelPadding'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['labelMargin'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['ticksWithoutCollisions'] = 50 config['graph_type'] = "tracked_domain" config['types'] = {} regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$" regx = re.compile(regex_str) for tgroup in groups: group = tgroup.replace(REPLACE_CHAR, ".") data_type = "tpd" if group in zones: data_type = "tracked_domain" elif re.match(regx, group): data_type = "cidr" config['types'][tgroup] = { "short": group, "long": "A group from the network: " + group, "data_type": data_type } config['constraints'] = [] tmp = int(math.ceil(math.sqrt(len(groups)))) + 1 x = [] y = [] for i in range(1, tmp): val = round((i * 1.0) / tmp, 2) x.append(str(val)) y.append(str(val)) x_pos = 0 y_pos = 0 for group in groups: config['constraints'].append({ "has": { "type": group }, "type": "position", "x": x[x_pos], "y": y[y_pos] }) x_pos = x_pos + 1 if x_pos >= len(x): x_pos = 0 y_pos = y_pos + 1 config['jsonUrl'] = "/api/v1.0/graphs/" + zone new_data['config'] = config new_data['created'] = datetime.now() new_data['zone'] = zone new_docs_data = {} new_docs_data['docs'] = {} new_docs_data['zone'] = zone new_docs_data['created'] = datetime.now() new_graph_data = {} new_graph_data['data'] = {} for i in range(0, len(data['nodes'])): new_graph_data['data'][data['nodes'][i]['id'].replace( ".", REPLACE_CHAR)] = data['nodes'][i] new_docs_data['docs'][data['nodes'][i]['id'].replace( ".", REPLACE_CHAR)] = data['nodes'][i]['docs'] del new_graph_data['data'][data['nodes'][i]['id'].replace( ".", REPLACE_CHAR)]['docs'] new_graph_data['created'] = datetime.now() new_graph_data['zone'] = zone new_graph_data['directed'] = data['directed'] new_graph_data['multigraph'] = data['multigraph'] new_graph_data['errs'] = [] new_links_data = {} new_links_data['links'] = data['links'] new_links_data['created'] = datetime.now() new_links_data['zone'] = zone new_links_data['directed'] = data['directed'] new_links_data['multigraph'] = data['multigraph'] new_links_data['errs'] = [] try: graphs_collection.remove({'zone': zone}) graphs_collection.insert_one(new_data) graphs_data_collection.remove({'zone': zone}) graphs_data_collection.insert_one(new_graph_data) graphs_links_collection.remove({'zone': zone}) graphs_links_collection.insert_one(new_links_data) graphs_docs_collection.remove({'zone': zone}) graphs_docs_collection.insert_one(new_docs_data) except: logger.error("ERROR: Can't insert: " + zone) time.sleep(1) # Remove last week's old entries # In theory, shouldn't do anything but being complete lastweek = datetime.now() - timedelta(days=7) graphs_collection.remove({'created': {"$lt": lastweek}}) graphs_data_collection.remove({'created': {"$lt": lastweek}}) graphs_links_collection.remove({'created': {"$lt": lastweek}}) graphs_docs_collection.remove({'created': {"$lt": lastweek}}) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete")
def main(): """ Begin Main() """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() mongo_ct = mongo_connector.get_certificate_transparency_connection() cert_graphs_collection = mongo_connector.get_cert_graphs_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, 'create_cert_graphs') jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) parser = argparse.ArgumentParser(description='Creates and stores certificate graphs in the database based on one or more sources.') parser.add_argument('--check_censys', action='store_true', default=False, required=False, help='Whether to check the Censys collection in the database') parser.add_argument('--check_443_scans', action='store_true', default=False, required=False, help='Whether to check the zgrab collection in the database') parser.add_argument('--check_ct_scans', action='store_true', default=False, required=False, help='Whether to check the CT collection in the database') parser.add_argument('--zgrab_version', default=2, type=int, choices=[1, 2], metavar="version", help='The version of ZGrab used to collect data') args = parser.parse_args() if args.check_censys is True: censys_collection = mongo_connector.get_censys_connection() if args.check_443_scans is True: zgrab_collection = mongo_connector.get_zgrab_443_data_connection() for zone in zones: logger.info("Creating: " + zone) graph = nx.DiGraph() certs_list = {} if args.check_ct_scans: certs_list = get_current_ct_certificates(mongo_ct, zone) if args.check_censys: certs_list = add_censys_certificates(censys_collection, zone, certs_list) if args.check_443_scans: if args.zgrab_version == 1: certs_list = add_terminal_zgrab_certificates(zgrab_collection, zone, certs_list) certs_list = add_initial_zgrab_certificates(zgrab_collection, zone, certs_list) else: certs_list = add_terminal_zgrab2_certificates(zgrab_collection, zone, certs_list) certs_list = add_initial_zgrab2_certificates(zgrab_collection, zone, certs_list) graph = create_nodes(graph, mongo_connector, zone, certs_list) data = json_graph.node_link_data(graph) my_data = {} my_data['links'] = data['links'] my_data['nodes'] = data['nodes'] my_data['zone'] = zone my_data['created'] = datetime.now() cert_graphs_collection.remove({'zone': zone}) cert_graphs_collection.insert(my_data) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") dns_types = { "a": 1, "ns": 2, "cname": 5, "soa": 6, "ptr": 12, "hinfo": 13, "mx": 15, "txt": 16, "aaaa": 28, "srv": 33, "naptr": 35, "ds": 43, "rrsig": 46, "dnskey": 48, } mongo_connector = MongoConnector.MongoConnector() all_dns_collection = mongo_connector.get_all_dns_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, "marinus_dns") jobs_manager.record_job_start() dns_manager = DNSManager.DNSManager(mongo_connector) zones = ZoneManager.get_distinct_zones(mongo_connector) google_dns = GoogleDNS.GoogleDNS() for zone in zones: time.sleep(1) for dtype, dnum in dns_types.items(): result = google_dns.fetch_DNS_records(zone, dnum) if result == []: logger.debug("No records found for " + zone) else: new_record = result[0] new_record["status"] = "confirmed" new_record["zone"] = zone new_record["created"] = datetime.now() logger.debug("Found " + dtype + " for: " + zone) dns_manager.insert_record(new_record, "marinus") logger.info("Starting SOA Search") soa_searches = find_sub_zones(all_dns_collection) for entry in soa_searches: time.sleep(1) result = google_dns.fetch_DNS_records(zone, dns_types["soa"]) if result != []: new_record = result[0] new_record["status"] = "confirmed" new_record["zone"] = get_fld_from_value(entry, "") new_record["created"] = datetime.now() logger.debug("Found SOA: " + entry) if new_record["zone"] != "": dns_manager.insert_record(new_record, "marinus") jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) if is_running(os.path.basename(__file__)): logger.warning("Already running...") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") parser = argparse.ArgumentParser( description="Parse Sonar files based on CIDRs.") parser.add_argument( "--sonar_file_type", choices=["dns-any", "dns-a", "rdns"], required=True, help='Specify "dns-any", "dns-a", or "rdns"', ) parser.add_argument( "--database", choices=["local", "remote"], required=False, default="local", help="Whether to use the local or remote DB", ) args = parser.parse_args() r7 = Rapid7.Rapid7() if args.database == "remote": mongo_connection = RemoteMongoConnector.RemoteMongoConnector() dns_manager = DNSManager.DNSManager(mongo_connection, "get_sonar_data_dns") else: mongo_connection = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connection) ip_manager = IPManager.IPManager(mongo_connection) zones = ZoneManager.get_distinct_zones(mongo_connection) logger.info("Zone length: " + str(len(zones))) save_directory = "./files/" check_save_location(save_directory) # A session is necessary for the multi-step log-in process s = requests.Session() if args.sonar_file_type == "rdns": jobs_manager = JobsManager.JobsManager(mongo_connection, "get_data_by_cidr_rdns") jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "rdns", jobs_manager) if html_parser.rdns_url == "": logger.error("Unknown Error") jobs_manager.record_job_error() exit(0) unzipped_rdns = download_remote_files(logger, s, html_parser.rdns_url, save_directory, jobs_manager) update_rdns(logger, unzipped_rdns, mongo_connection, dns_manager, ip_manager, zones) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) logger.info("RDNS Complete") jobs_manager.record_job_complete() elif args.sonar_file_type == "dns": jobs_manager = JobsManager.JobsManager(mongo_connection, "get_data_by_cidr_dns") jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.any_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.any_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, dns_manager, ip_manager, zones) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) elif args.sonar_file_type == "dns-a": jobs_manager = JobsManager.JobsManager(mongo_connection, "get_data_by_cidr_dns-a") jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.a_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.a_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, dns_manager, ip_manager, zones) if html_parser.aaaa_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.aaaa_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, dns_manager, ip_manager, zones) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) logger.info("DNS Complete") jobs_manager.record_job_complete() else: logger.error("Unrecognized sonar_file_type option. Exiting...") now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") azure_connector = AzureConnector.AzureConnector() mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zone_ingestor = ZoneIngestor.ZoneIngestor() jobs_manager = JobsManager.JobsManager(mongo_connector, "fetch_azure_dns") jobs_manager.record_job_start() current_zones = ZoneManager.get_distinct_zones(mongo_connector) resource_client = azure_connector.get_resources_client() resources = [] # The resource list is not currently used. for item in resource_client.resource_groups.list(): resources.append(item.name) dns_client = azure_connector.get_dns_client() zones = dns_client.zones.list() # The type of records the Azure DNS will let you configure record_types = { "A": "arecords", "AAAA": "aaaa_records", "MX": "mx_records", "NS": "ns_records", "PTR": "ptr_records", "SRV": "srv_records", "TXT": "txt_records", "CNAME": "cname_record", "SOA": "soa_record", } for zone in zones: logger.info("Zone: " + zone.name) data = split_id(zone.id) if zone.zone_type == ZoneType.public: logger.info(zone.name + " is public:") if zone.name not in current_zones: logger.debug("Creating zone: " + zone.name) zone_ingestor.add_zone(zone.name, "azure:" + data["resourceGroups"]) try: logger.info("ResourceGroup: " + data["resourceGroups"]) records = dns_client.record_sets.list_all_by_dns_zone( data["resourceGroups"], zone.name ) for entry in records: # The record_data id value ends in rtype/rvalue so you must guess the rtype record_data = split_id(entry.id) for rtype in record_types: if rtype in record_data: results = extract_record_set_value(logger, rtype, entry) for result in results: result["zone"] = zone.name result["created"] = datetime.now() result["status"] = "confirmed" dns_manager.insert_record( result, "azure:" + data["resourceGroups"] ) except: logger.warning("No records found") jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Make database connections mongo_connector = MongoConnector.MongoConnector() ct_collection = mongo_connector.get_certificate_transparency_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, "facebook_certs") jobs_manager.record_job_start() file_path = "/mnt/workspace/ct_facebook/" fb_connector = FacebookConnector.FacebookConnector() access_token = fb_connector.get_facebook_access_token() zones = ZoneManager.get_distinct_zones(mongo_connector) x509_parser = X509Parser.X509Parser() parser = argparse.ArgumentParser( description="Download DNS and/or certificate information from crt.sh.") parser.add_argument( "--fetch_cert_records", choices=["dbAndSave", "dbOnly"], default="dbAndSave", help= "Indicates whether to download the raw files or just record in the database", ) parser.add_argument( "--cert_save_location", required=False, default=file_path, help= "Indicates where to save the certificates on disk when choosing dbAndSave", ) args = parser.parse_args() check_save_location(args.cert_save_location) save_location = args.cert_save_location if not save_location.endswith("/"): save_location = save_location + "/" for zone in zones: time.sleep(15) results = fetch_domain(logger, jobs_manager, fb_connector, access_token, zone) if results is None: logger.warning("ERROR looking up: " + zone) continue logger.info(zone + ": " + str(len(results))) for result in results: if args.fetch_cert_records == "dbAndSave": cert_f = open( save_location + zone + "_" + result["id"] + ".pem", "w") cert_f.write(result["certificate_pem"]) cert_f.close() cert = x509_parser.parse_data(result["certificate_pem"], "facebook") cert["facebook_id"] = result["id"] if (ct_collection.count_documents( {"fingerprint_sha256": cert["fingerprint_sha256"]}) == 0): mongo_connector.perform_insert(ct_collection, cert) else: if (ct_collection.count_documents({ "fingerprint_sha256": cert["fingerprint_sha256"], "facebook_id": result["id"], "zones": zone, }) == 0): ct_collection.update_one( {"fingerprint_sha256": cert["fingerprint_sha256"]}, { "$set": { "marinus_updated": datetime.now(), "facebook_id": result["id"], }, "$addToSet": { "zones": zone }, }, ) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "extract_mx_domains") google_dns = GoogleDNS.GoogleDNS() jobs_manager.record_job_start() dns_names = [] round_two = [] zones = ZoneManager.get_distinct_zones(mongo_connector) # Collect the list of domains from the MX Records extract_mx_names(dns_names, dns_manager) input_list = [] # Some MX records point to the third-party domains. # Therefore, we filter to only the root domains that belong to the tracked company. logger.info("Pre-filter list: " + str(len(dns_names))) for hostname in dns_names: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) # Pause to prevent DoS-ing of Google's HTTPS DNS Service time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr["fqdn"], zones) if temp_zone is not None: record = {"fqdn": ip_addr["fqdn"]} record["zone"] = temp_zone record["created"] = datetime.now() record["type"] = ip_addr["type"] record["value"] = ip_addr["value"] record["status"] = "unknown" input_list.append(record) if ip_addr["type"] == "cname" and is_tracked_zone( ip_addr["value"], zones ): add_to_round_two(ip_addr["value"], round_two) else: logger.warning("Failed IP Lookup for: " + hostname) else: logger.warning("Failed match on zone for: " + hostname) dead_dns_collection = mongo_connector.get_dead_dns_connection() # Some DNS records will be CNAME records pointing to other tracked domains. # This is a single level recursion to lookup those domains. logger.info("Round Two list: " + str(len(round_two))) for hostname in round_two: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr["fqdn"], zones) if temp_zone is not None: record = {"fqdn": ip_addr["fqdn"]} record["zone"] = temp_zone record["created"] = datetime.now() record["type"] = ip_addr["type"] record["value"] = ip_addr["value"] record["status"] = "unknown" input_list.append(record) else: logger.warning("Failed IP Lookup for: " + hostname) original_record = dns_manager.find_one({"fqdn": hostname}, "mx") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) else: logger.warning("Failed match on zone for: " + hostname) # Record all the results. dns_manager.remove_by_source("mx") logger.info("List length: " + str(len(input_list))) for final_result in input_list: dns_manager.insert_record(final_result, "mx") # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ global global_exit_flag global global_zgrab_path logger = LoggingUtil.create_log(__name__) parser = argparse.ArgumentParser( description="Launch zgrab against domains using port 80 or 443.") parser.add_argument("-p", choices=["443", "80"], metavar="port", help="The web port: 80 or 443") parser.add_argument("-t", default=5, type=int, metavar="threadCount", help="The number of threads") parser.add_argument( "--zgrab_path", default=global_zgrab_path, metavar="zgrabVersion", help="The version of ZGrab to use", ) args = parser.parse_args() if args.p == None: logger.error("A port value (80 or 443) must be provided.") exit(1) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() logger.warning(str(now) + ": I am already running! Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") rm_connector = RemoteMongoConnector.RemoteMongoConnector() all_dns_collection = rm_connector.get_all_dns_connection() jobs_manager = JobsManager.JobsManager(rm_connector, "zgrab_http_domain-" + args.p) jobs_manager.record_job_start() if args.p == "443": zgrab_collection = rm_connector.get_zgrab_443_data_connection() run_command = run_port_443_command else: zgrab_collection = rm_connector.get_zgrab_80_data_connection() run_command = run_port_80_command check_save_location("./json_p" + args.p) global_zgrab_path = args.zgrab_path zones = ZoneManager.get_distinct_zones(rm_connector) ip_manager = IPManager.IPManager(rm_connector) for zone in zones: global_exit_flag = 0 domains = get_domains(all_dns_collection, ip_manager, zone) if len(domains) == 0: continue num_threads = args.t if len(domains) < args.t: num_threads = len(domains) logger.debug("Creating " + str(num_threads) + " threads") threads = [] for thread_id in range(1, num_threads + 1): thread = ZgrabThread( thread_id, global_work_queue, args.p, run_command, zone, zgrab_collection, ) thread.start() threads.append(thread) thread_id += 1 logger.debug(zone + " length: " + str(len(domains))) logger.info("Populating Queue") global_queue_lock.acquire() for domain in domains: global_work_queue.put(domain) global_queue_lock.release() # Wait for queue to empty while not global_work_queue.empty(): pass logger.info("Queue empty") # Notify threads it's time to exit global_exit_flag = 1 # Wait for all threads to complete for t in threads: t.join() # Remove last week's old entries lastweek = datetime.now() - timedelta(days=7) zgrab_collection.delete_many({ "domain": { "$ne": "<nil>" }, "timestamp": { "$lt": lastweek } }) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ if is_running(os.path.basename(__file__)): print("Already running...") exit(0) now = datetime.now() print("Starting: " + str(now)) r7 = Rapid7.Rapid7() zones = ZoneManager.get_distinct_zones(mongo_connector) parser = argparse.ArgumentParser( description='Parse Sonar files based on domain zones.') parser.add_argument('--sonar_file_type', required=True, help='Specify "dns-any", "dns-a", or "rdns"') args = parser.parse_args() jobs_collection = mongo_connector.get_jobs_connection() # A session is necessary for the multi-step log-in process s = requests.Session() if args.sonar_file_type == "rdns": now = datetime.now() print("Updating RDNS: " + str(now)) jobs_collection.update_one({'job_name': 'get_sonar_data_rdns'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'RUNNING' } }) try: html_parser = r7.find_file_locations(s, "rdns", "get_sonar_data_rdns", jobs_collection) if html_parser.rdns_url == "": now = datetime.now() print("Unknown Error: " + str(now)) jobs_collection.update_one({'job_name': 'get_sonar_data_rdns'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'ERROR' } }) exit(0) unzipped_rdns = download_remote_files(s, html_parser.rdns_url, global_data_dir, "get_sonar_data_rdns", jobs_collection) update_rdns(unzipped_rdns, zones, mongo_connector) except Exception as ex: now = datetime.now() print("Unknown error occured at: " + str(now)) print("Unexpected error: " + str(ex)) jobs_collection.update_one({'job_name': 'get_sonar_data_rdns'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'ERROR' } }) exit(0) # Update used instead of update_one due to old pymongo library on IT machines jobs_collection.update_one({'job_name': 'get_sonar_data_rdns'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) elif args.sonar_file_type == "dns-any": now = datetime.now() print("Updating DNS: " + str(now)) jobs_collection.update_one({'job_name': 'get_sonar_data_dns-any'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'RUNNING' } }) try: html_parser = r7.find_file_locations(s, "fdns", "get_sonar_data_dns-any", jobs_collection) if html_parser.any_url != "": unzipped_dns = download_remote_files(s, html_parser.any_url, global_data_dir, "get_sonar_data_dns-any", jobs_collection) update_dns(unzipped_dns, zones, global_dns_manager) except Exception as ex: now = datetime.now() print("Unknown error occured at: " + str(now)) print("Unexpected error: " + str(ex)) jobs_collection.update_one({'job_name': 'get_sonar_data_dns-any'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'ERROR' } }) exit(0) # Update used instead of update_one due to old pymongo library on IT machines jobs_collection.update_one({'job_name': 'get_sonar_data_dns-any'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) elif args.sonar_file_type == "dns-a": now = datetime.now() print("Updating DNS: " + str(now)) jobs_collection.update_one({'job_name': 'get_sonar_data_dns-a'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'RUNNING' } }) try: html_parser = r7.find_file_locations(s, "fdns", "get_sonar_data_dns-a", jobs_collection) if html_parser.a_url != "": unzipped_dns = download_remote_files(s, html_parser.a_url, global_data_dir, "get_sonar_data_dns-a", jobs_collection) update_dns(unzipped_dns, zones, global_dns_manager) if html_parser.aaaa_url != "": unzipped_dns = download_remote_files(s, html_parser.aaaa_url, global_data_dir, "get_sonar_data_dns-a", jobs_collection) update_dns(unzipped_dns, zones, global_dns_manager) except Exception as ex: now = datetime.now() print("Unknown error occured at: " + str(now)) print("Unexpected error: " + str(ex)) jobs_collection.update_one({'job_name': 'get_sonar_data_dns-a'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'ERROR' } }) exit(0) # Update used instead of update_one due to old pymongo library on IT machines jobs_collection.update_one({'job_name': 'get_sonar_data_dns-a'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) else: print("Unrecognized sonar_file_type option. Exiting...") now = datetime.now() print("Complete: " + str(now))
def main(): """ The main thread for this program. """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() jobs_collection = mongo_connector.get_jobs_connection() zones = ZoneManager.get_distinct_zones(mongo_connector) tpds = get_tpds(mongo_connector) # For third-party-domain in the list of third-party-domains for tpd in tpds: groups = [] graph = nx.DiGraph() add_to_list(tpd, groups) # A space is added because sometimes the tpd is the same as the end target node graph.add_node(tpd + " ", data_type="tld", type=0, depends=[], dependedOnBy=[], docs="<h1>Parent</h1>") # Get the zones associated with the tpd find_zones_by_tld(graph, tpd, groups, mongo_connector) data = json_graph.node_link_data(graph) reformat_data(data, tpd, groups) new_data = {} new_data['directed'] = data['directed'] new_data['graph'] = data['graph'] new_data['multigraph'] = data['multigraph'] new_data['errs'] = [] new_data['links'] = data['links'] new_data['data'] = {} for i in range(0, len(data['nodes'])): new_data['data'][data['nodes'][i]['id'].replace( ".", REPLACE_CHAR)] = data['nodes'][i] for entry in new_data['data']: for dep in new_data['data'][entry]['depends']: if new_data['data'][entry]['name'] not in new_data['data'][ dep.replace(".", REPLACE_CHAR)]['dependedOnBy']: new_data['data'][dep.replace( ".", REPLACE_CHAR)]['dependedOnBy'].append( new_data['data'][entry]['name']) for dep in new_data['data'][entry]['dependedOnBy']: if new_data['data'][entry]['name'] not in new_data['data'][ dep.replace(".", REPLACE_CHAR)]['depends']: new_data['data'][dep.replace( ".", REPLACE_CHAR)]['depends'].append( new_data['data'][entry]['name']) for entry in new_data['data']: new_data['data'][entry]['docs'] = build_docs( new_data['data'][entry], tpd, groups) config = {} config['title'] = tpd + " Network Map" config['graph'] = {} config['graph']['linkDistance'] = 150 config['graph']['charge'] = -400 config['graph']['height'] = 800 config['graph']['numColors'] = len(groups) config['graph']['labelPadding'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['labelMargin'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['ticksWithoutCollisions'] = 50 config['graph_type'] = "tpd" config['types'] = {} regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$" regx = re.compile(regex_str) for tgroup in groups: data_type = "tpd" group = tgroup.replace(REPLACE_CHAR, ".") if group in zones: data_type = "tracked_domain" elif re.match(regx, group): data_type = "cidr" config['types'][tgroup] = { "short": group, "long": "A group from the network: " + group, "data_type": data_type } config['constraints'] = [] tmp = int(math.ceil(math.sqrt(len(groups)))) + 1 x = [] y = [] for i in range(1, tmp): val = round((i * 1.0) / tmp, 2) x.append(str(val)) y.append(str(val)) x_pos = 0 y_pos = 0 for group in groups: config['constraints'].append({ "has": { "type": group }, "type": "position", "x": x[x_pos], "y": y[y_pos] }) x_pos = x_pos + 1 if x_pos >= len(x): x_pos = 0 y_pos = y_pos + 1 config['jsonUrl'] = "/api/v1.0/tpd_graphs/" + tpd new_data['config'] = config new_data['created'] = datetime.now() new_data['zone'] = tpd tpd_graphs_collection = mongo_connector.get_tpd_graphs_connection() tpd_graphs_collection.remove({'zone': tpd}) try: tpd_graphs_collection.insert_one(new_data) except: print("ERROR: Could not insert " + tpd) time.sleep(1) # Remove last week's old entries lastweek = datetime.now() - timedelta(days=7) tpd_graphs_collection.remove({'created': {"$lt": lastweek}}) # Record status jobs_collection.update_one({'job_name': 'create_tpd_graphs'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) now = datetime.now() print("Complete: " + str(now))
def main(): """ The main thread for this program. """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() jobs_manager = JobsManager.JobsManager(mongo_connector, "create_tpd_graphs") jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) tpds = get_tpds(mongo_connector) # For third-party-domain in the list of third-party-domains for tpd in tpds: groups = [] graph = nx.DiGraph() add_to_list(tpd, groups) # A space is added because sometimes the tpd is the same as the end target node graph.add_node( tpd + " ", data_type="tld", type=0, depends=[], dependedOnBy=[], docs="<h1>Parent</h1>", ) # Get the zones associated with the tpd find_zones_by_tld(graph, tpd, groups, mongo_connector) data = json_graph.node_link_data(graph) reformat_data(data, tpd, groups) new_data = {} new_data["directed"] = data["directed"] new_data["graph"] = data["graph"] new_data["multigraph"] = data["multigraph"] new_data["errs"] = [] new_data["links"] = data["links"] new_data["data"] = {} for i in range(0, len(data["nodes"])): new_data["data"][data["nodes"][i]["id"].replace( ".", REPLACE_CHAR)] = data["nodes"][i] for entry in new_data["data"]: for dep in new_data["data"][entry]["depends"]: if (new_data["data"][entry]["name"] not in new_data["data"][dep.replace( ".", REPLACE_CHAR)]["dependedOnBy"]): new_data["data"][dep.replace( ".", REPLACE_CHAR)]["dependedOnBy"].append( new_data["data"][entry]["name"]) for dep in new_data["data"][entry]["dependedOnBy"]: if (new_data["data"][entry]["name"] not in new_data["data"][dep.replace( ".", REPLACE_CHAR)]["depends"]): new_data["data"][dep.replace( ".", REPLACE_CHAR)]["depends"].append( new_data["data"][entry]["name"]) for entry in new_data["data"]: new_data["data"][entry]["docs"] = build_docs( new_data["data"][entry], tpd, groups) config = {} config["title"] = tpd + " Network Map" config["graph"] = {} config["graph"]["linkDistance"] = 150 config["graph"]["charge"] = -400 config["graph"]["height"] = 800 config["graph"]["numColors"] = len(groups) config["graph"]["labelPadding"] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config["graph"]["labelMargin"] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config["graph"]["ticksWithoutCollisions"] = 50 config["graph_type"] = "tpd" config["types"] = {} regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$" regx = re.compile(regex_str) for tgroup in groups: data_type = "tpd" group = tgroup.replace(REPLACE_CHAR, ".") if group in zones: data_type = "tracked_domain" elif re.match(regx, group): data_type = "cidr" config["types"][tgroup] = { "short": group, "long": "A group from the network: " + group, "data_type": data_type, } config["constraints"] = [] tmp = int(math.ceil(math.sqrt(len(groups)))) + 1 x = [] y = [] for i in range(1, tmp): val = round((i * 1.0) / tmp, 2) x.append(str(val)) y.append(str(val)) x_pos = 0 y_pos = 0 for group in groups: config["constraints"].append({ "has": { "type": group }, "type": "position", "x": x[x_pos], "y": y[y_pos], }) x_pos = x_pos + 1 if x_pos >= len(x): x_pos = 0 y_pos = y_pos + 1 config["jsonUrl"] = "/api/v1.0/tpd_graphs/" + tpd new_data["config"] = config new_data["created"] = datetime.now() new_data["zone"] = tpd tpd_graphs_collection = mongo_connector.get_tpd_graphs_connection() tpd_graphs_collection.delete_one({"zone": tpd}) try: tpd_graphs_collection.insert_one(new_data) except: logger.error("ERROR: Could not insert " + tpd) time.sleep(1) # Remove last week's old entries lastweek = datetime.now() - timedelta(days=7) tpd_graphs_collection.delete_many({"created": {"$lt": lastweek}}) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() all_dns_collection = mongo_connector.get_all_dns_connection() dns_manager = DNSManager.DNSManager(mongo_connector) GDNS = GoogleDNS.GoogleDNS() ip_manager = IPManager.IPManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'remove_expired_entries') jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) # The sources for which to remove expired entries results = mongo_connector.perform_distinct(all_dns_collection, 'sources.source') sources = [] for source in results: temp = {} temp['name'] = source if "common_crawl" in source: temp['diff'] = -4 else: temp['diff'] = -2 sources.append(temp) # Before completely removing old entries, make an attempt to see if they are still valid. # Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source. # Rather than throw away valid information, this will archive it. for entry in sources: removal_date = monthdelta(datetime.now(), entry['diff']) source = entry['name'] logger.debug("Removing " + source + " as of: " + str(removal_date)) last_domain = "" results = all_dns_collection.find({ 'sources': { "$size": 1 }, 'sources.source': source, 'sources.updated': { "$lt": removal_date } }) for result in results: if result['fqdn'] != last_domain: last_domain = result['fqdn'] lookup_int = get_lookup_int(logger, result, GDNS) dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int) if dns_result != []: insert_current_results(dns_result, dns_manager, zones, result, source) dns_manager.remove_all_by_source_and_date(source, entry['diff']) # Get the date for today minus two months d_minus_2m = monthdelta(datetime.now(), -2) logger.info("Removing SRDNS as of: " + str(d_minus_2m)) # Remove the old records srdns_collection = mongo_connector.get_sonar_reverse_dns_connection() srdns_collection.remove({'updated': {"$lt": d_minus_2m}}) ip_manager.delete_records_by_date(d_minus_2m) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) if is_running(os.path.basename(__file__)): logger.warning("Already running...") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") r7 = Rapid7.Rapid7() zones = ZoneManager.get_distinct_zones(mongo_connector) save_directory = "./files/" parser = argparse.ArgumentParser( description='Parse Sonar files based on domain zones.') parser.add_argument('--sonar_file_type', required=True, help='Specify "dns-any", "dns-a", or "rdns"') args = parser.parse_args() check_save_location(save_directory) # A session is necessary for the multi-step log-in process s = requests.Session() if args.sonar_file_type == "rdns": logger.info("Updating RDNS") jobs_manager = JobsManager.JobsManager(mongo_connector, 'get_sonar_data_rdns') jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "rdns", jobs_manager) if html_parser.rdns_url == "": logger.error("Unknown Error") jobs_manager.record_job_error() exit(0) unzipped_rdns = download_remote_files(logger, s, html_parser.rdns_url, save_directory, jobs_manager) update_rdns(logger, unzipped_rdns, zones, mongo_connector) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) jobs_manager.record_job_complete() elif args.sonar_file_type == "dns-any": logger.info("Updating DNS") jobs_manager = JobsManager.JobsManager(mongo_connector, 'get_sonar_data_dns-any') jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.any_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.any_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, global_dns_manager) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) jobs_manager.record_job_complete() elif args.sonar_file_type == "dns-a": logger.info("Updating DNS") jobs_manager = JobsManager.JobsManager(mongo_connector, 'get_sonar_data_dns-a') jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.a_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.a_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, global_dns_manager) if html_parser.aaaa_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.aaaa_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, global_dns_manager) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) jobs_manager.record_job_complete() else: logger.error("Unrecognized sonar_file_type option. Exiting...") now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Connect to the remote databases mongo_connector = MongoConnector.MongoConnector() rm_connector = RemoteMongoConnector.RemoteMongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zones = ZoneManager.get_distinct_zones(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "remote_download") jobs_manager.record_job_start() remote_jobs_collection = rm_connector.get_jobs_connection() # Check the status of the Censys job on the remote database try: status = remote_jobs_collection.find_one({"job_name": "censys"}) except: logger.error("Can not connect to remote database") jobs_manager.record_job_error() exit(1) if (status is not None and "status" in status and status["status"] != jobs_manager.COMPLETE): logger.info("Censys scans status is not COMPLETE") elif (status is not None and "status" in status and status["status"] == jobs_manager.COMPLETE): # Get connections to the relevant collections. censys_collection = mongo_connector.get_zgrab_443_data_connection() remote_censys_collection = rm_connector.get_zgrab_443_data_connection() download_censys_scan_info(logger, censys_collection, remote_censys_collection) # Tell the remote database that is safe to start processing the next Censys file remote_jobs_collection.update_one( {"job_name": "censys"}, { "$currentDate": { "updated": True }, "$set": { "status": jobs_manager.READY } }, ) # Get connections to the relevant HTTPS collections. zgrab_443_data_collection = mongo_connector.get_zgrab_443_data_connection() remote_zgrab_443_data_collection = rm_connector.get_zgrab_443_data_connection( ) download_zgrab_info(logger, zgrab_443_data_collection, remote_zgrab_443_data_collection) # Get connections to the relevant HTTP collections. zgrab_80_data_collection = mongo_connector.get_zgrab_80_data_connection() remote_zgrab_80_data_collection = rm_connector.get_zgrab_80_data_connection( ) download_zgrab_info(logger, zgrab_80_data_collection, remote_zgrab_80_data_collection) # Get connections to the relevant port collections. zgrab_port_data_collection = mongo_connector.get_zgrab_port_data_connection( ) remote_zgrab_port_data_collection = rm_connector.get_zgrab_port_data_connection( ) download_zgrab_port_info(logger, zgrab_port_data_collection, remote_zgrab_port_data_collection) # Download latest whois information status = remote_jobs_collection.find_one({"job_name": "whois_lookups"}) if status["status"] == jobs_manager.COMPLETE: whois_collection = mongo_connector.get_whois_connection() remote_whois_collection = rm_connector.get_whois_connection() download_whois_data(logger, whois_collection, remote_whois_collection) remote_jobs_collection.update_one( {"job_name": "whois"}, {"$set": { "status": jobs_manager.READY }}) # Download Amass results amass_collection = mongo_connector.get_owasp_amass_connection() remote_amass_collection = rm_connector.get_owasp_amass_connection() download_amass_data(logger, amass_collection, remote_amass_collection, dns_manager, zones) # Download the status of the remote jobs download_jobs_status(logger, jobs_manager._jobs_collection, remote_jobs_collection) # Download remote sonar DNS findings download_sonar_dns(logger, dns_manager, rm_connector) # Download remote sonar RDNS findings download_sonar_rdns(logger, mongo_connector, rm_connector) # Update the local jobs database to done jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Set up the common objects mongo_connector = MongoConnector.MongoConnector() ct_collection = mongo_connector.get_certificate_transparency_connection() zones = ZoneManager.get_distinct_zones(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "get_crt_sh") jobs_manager.record_job_start() save_location = "/mnt/workspace/crt_sh" download_method = 'dbAndSave' parser = argparse.ArgumentParser( description='Download DNS and/or certificate information from crt.sh.') parser.add_argument( '--fetch_dns_records', action='store_true', help='Indicates whether to add DNS entries to the database') parser.add_argument( '--download_methods', choices=['dbAndSave', 'dbOnly'], default=download_method, help= 'Indicates whether to download the raw files or just record in the database.' ) parser.add_argument( '--cert_save_location', required=False, default=save_location, help= 'Indicates where to save the certificates on disk when choosing dbAndSave' ) args = parser.parse_args() if args.cert_save_location: save_location = args.cert_save_location if not save_location.endswith("/"): save_location = save_location + "/" if args.download_methods == 'dbAndSave': check_save_location(save_location) for zone in zones: # Pace out requests so as not to DoS crt.sh and Google DNS time.sleep(5) # This could be done with backoff but we don't want to be overly aggressive. json_result = make_https_request( logger, "https://crt.sh/?q=%25." + zone + "&output=json") if json_result is None: logger.warning("Can't find result for: " + zone) json_result = "{}" json_data = json.loads(json_result) new_names = [] new_ids = [] for entry in json_data: if entry['min_cert_id'] not in new_ids: new_ids.append(entry['min_cert_id']) if "*" not in entry["name_value"] and entry[ "name_value"] not in new_names: new_names.append(entry["name_value"]) if args.fetch_dns_records: add_new_domain_names(new_names, zones, mongo_connector) if args.download_methods == "dbAndSave": add_new_certificate_values(logger, new_ids, ct_collection, zones, save_location) elif args.download_methods == "dbOnly": add_new_certificate_values(logger, new_ids, ct_collection, zones, None) # Set isExpired for any entries that have recently expired. ct_collection.update( { "not_after": { "$lt": datetime.utcnow() }, "isExpired": False }, {"$set": { "isExpired": True }}, multi=True) jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) google_dns = GoogleDNS.GoogleDNS() jobs_collection = mongo_connector.get_jobs_connection() dns_names = [] round_two = [] zones = ZoneManager.get_distinct_zones(mongo_connector) # Collect the list of domains from the MX Records extract_mx_names(dns_names, dns_manager) input_list = [] # Some MX records point to the third-party domains. # Therefore, we filter to only the root domains that belong to the tracked company. print("Pre-filter list: " + str(len(dns_names))) for hostname in dns_names: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) # Pause to prevent DoS-ing of Google's HTTPS DNS Service time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) if ip_addr['type'] == "cname" and is_tracked_zone( ip_addr['value'], zones): add_to_round_two(ip_addr['value'], round_two) else: print("Failed IP Lookup for: " + hostname) else: print("Failed match on zone for: " + hostname) dead_dns_collection = mongo_connector.get_dead_dns_connection() # Some DNS records will be CNAME records pointing to other tracked domains. # This is a single level recursion to lookup those domains. print("Round Two list: " + str(len(round_two))) for hostname in round_two: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) else: print("Failed IP Lookup for: " + hostname) original_record = dns_manager.find_one({"fqdn": hostname}, "mx") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) else: print("Failed match on zone for: " + hostname) # Record all the results. dns_manager.remove_by_source("mx") print("List length: " + str(len(input_list))) for final_result in input_list: dns_manager.insert_record(final_result, "mx") # Record status jobs_collection.update_one({'job_name': 'extract_mx_domains'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) now = datetime.now() print("Ending: " + str(now))
def main(): now = datetime.now() print("Starting: " + str(now)) azure_connector = AzureConnector.AzureConnector() mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zone_ingestor = ZoneIngestor.ZoneIngestor() current_zones = ZoneManager.get_distinct_zones(mongo_connector) resource_client = azure_connector.get_resources_client() resources = [] # The resource list is not currently used. for item in resource_client.resource_groups.list(): resources.append(item.name) dns_client = azure_connector.get_dns_client() zones = dns_client.zones.list() # The type of records the Azure DNS will let you configure record_types = { 'A': 'arecords', 'AAAA': 'aaaa_records', 'MX': 'mx_records', 'NS': 'ns_records', 'PTR': 'ptr_records', 'SRV': 'srv_records', 'TXT': 'txt_records', 'CNAME': 'cname_record', 'SOA': 'soa_record' } for zone in zones: print("Zone: " + zone.name) data = split_id(zone.id) if zone.zone_type == ZoneType.public: print(zone.name + " is public:") if zone.name not in current_zones: print("Creating zone: " + zone.name) zone_ingestor.add_zone(zone.name, "azure:" + data["resourceGroups"]) try: print("ResourceGroup: " + data["resourceGroups"]) records = dns_client.record_sets.list_all_by_dns_zone( data["resourceGroups"], zone.name) for entry in records: # The record_data id value ends in rtype/rvalue so you must guess the rtype record_data = split_id(entry.id) for rtype in record_types: if rtype in record_data: results = extract_record_set_value(rtype, entry) for result in results: result['zone'] = zone.name result['created'] = datetime.now() result['status'] = 'confirmed' dns_manager.insert_record( result, "azure:" + data["resourceGroups"]) except: print("No records found")
def main(): """ Begin main... """ logger = LoggingUtil.create_log(__name__) mongo_connector = MongoConnector.MongoConnector() now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") jobs_manager = JobsManager.JobsManager(mongo_connector, 'owasp_amass') zones = ZoneManager.get_distinct_zones(mongo_connector) dns_manager = DNSManager.DNSManager(mongo_connector) output_dir = "./amass_files/" arg_parser = argparse.ArgumentParser( description= 'Run the OWASP Amass tool and store the results in the database.') arg_parser.add_argument( '--config_file', required=False, help='An optional Amass config file. Otherwise, defaults will be used.' ) arg_parser.add_argument('--amass_path', required=True, help='The path to the amass binary') arg_parser.add_argument('--output_dir', default=output_dir, help="The path where to save Amass files.") arg_parser.add_argument('--amass_version', type=int, default=3, help='The version of OWASP Amass being used.') arg_parser.add_argument( '--sleep', type=int, default=5, help= 'Sleep time in seconds between amass runs so as not to overuse service limits.' ) args = arg_parser.parse_args() if not os.path.isfile(args.amass_path): logger.error("Incorrect amass_path argument provided") exit(1) if 'config_file' in args and not os.path.isfile(args.config_file): logger.error("Incorrect config_file location") exit(1) if 'output_dir' in args: output_dir = args.output_dir if not output_dir.endswith("/"): output_dir = output_dir + "/" check_save_location(output_dir) jobs_manager.record_job_start() # If the job died half way through, you can skip over domains that were already processed # when you restart the script. new_zones = [] for zone in zones: if not os.path.isfile(output_dir + zone + "-do.json"): new_zones.append(zone) for zone in new_zones: # Pace out calls to the Amass services time.sleep(args.sleep) command_line = [] command_line.append(args.amass_path) if int(args.amass_version) >= 3: command_line.append("enum") if args.config_file: command_line.append("-config") command_line.append(args.config_file) command_line.append("-d") command_line.append(zone) command_line.append("-src") command_line.append("-ip") command_line.append("-o") command_line.append(output_dir + zone + "-do.json") try: subprocess.check_call(command_line) except subprocess.CalledProcessError as e: # Even when there is an error, there will likely still be results. # We can continue with the data that was collected thus far. logger.warning("ERROR: Amass run exited with a non-zero status: " + str(e)) if os.path.isfile(output_dir + zone + "-do.json"): output = open(output_dir + zone + "-do.json", "r") json_data = [] for line in output: try: json_data.append(json.loads(line)) except: logger.warning("Amass wrote an incomplete line: " + str(line)) output.close() for finding in json_data: if 'type' in finding and finding[ 'type'] == 'infrastructure' or finding[ 'type'] == 'domain': # Not currently recording continue elif is_tracked_zone(finding['domain'], zones): record_finding(dns_manager, finding) else: # logger.debug("Skipping: " + finding['domain'] + " type: " + finding['type']) pass jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "extract_vt_domains") google_dns = GoogleDNS.GoogleDNS() jobs_manager.record_job_start() round_two = [] zones = ZoneManager.get_distinct_zones(mongo_connector) vt_collection = mongo_connector.get_virustotal_connection() vt_results = vt_collection.find({ "subdomains": { "$exists": True } }, { "zone": 1, "subdomains": 1 }).batch_size(30) input_list = [] # For each result found in the first pass across VirusTotal for result in vt_results: # Pause to prevent DoS-ing of Google's HTTPS DNS Service time.sleep(1) for hostname in result["subdomains"]: ips = google_dns.fetch_DNS_records(hostname) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr["fqdn"], zones) if temp_zone is not None: record = {"fqdn": ip_addr["fqdn"]} record["zone"] = temp_zone record["created"] = datetime.now() record["type"] = ip_addr["type"] record["value"] = ip_addr["value"] record["status"] = "unknown" input_list.append(record) if ip_addr["type"] == "cname" and is_tracked_zone( ip_addr["value"], zones): add_to_list(ip_addr["value"], round_two) else: logger.warning("Failed IP Lookup for: " + hostname) dead_dns_collection = mongo_connector.get_dead_dns_connection() # For each tracked CName result found in the first pass across VirusTotal logger.info("Round Two length: " + str(len(round_two))) for hostname in round_two: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr["fqdn"], zones) if temp_zone is not None: record = {"fqdn": ip_addr["fqdn"]} record["zone"] = temp_zone record["created"] = datetime.now() record["type"] = ip_addr["type"] record["value"] = ip_addr["value"] record["status"] = "unknown" input_list.append(record) else: original_record = dns_manager.find_one({"fqdn": hostname}, "virustotal") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) logger.warning("Failed IP Lookup for: " + hostname) else: logger.warning("Failed match on zone for: " + hostname) # Update the database dns_manager.remove_by_source("virustotal") logger.info("List length: " + str(len(input_list))) for final_result in input_list: dns_manager.insert_record(final_result, "virustotal") # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ if is_running(os.path.basename(__file__)): print("Already running...") exit(0) now = datetime.now() print("Starting: " + str(now)) r7 = Rapid7.Rapid7() mongo_connection = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connection) ip_manager = IPManager.IPManager(mongo_connection) rdns_collection = mongo_connection.get_sonar_reverse_dns_connection() zones = ZoneManager.get_distinct_zones(mongo_connection) print ("Zone length: " + str(len(zones))) save_directory = "./files/" parser = argparse.ArgumentParser(description='Parse Sonar files based on CIDRs.') parser.add_argument('--sonar_file_type', required=True, help='Specify "dns" or "rdns"') args = parser.parse_args() check_save_location(save_directory) # A session is necessary for the multi-step log-in process s = requests.Session() if args.sonar_file_type == "rdns": jobs_manager = JobsManager.JobsManager(mongo_connection, 'get_data_by_cidr_rdns') jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "rdns", jobs_manager) if html_parser.rdns_url == "": now = datetime.now() print ("Unknown Error: " + str(now)) jobs_manager.record_job_error() exit(0) unzipped_rdns = download_remote_files(s, html_parser.rdns_url, save_directory, jobs_manager) update_rdns(unzipped_rdns, rdns_collection, dns_manager, ip_manager, zones) except Exception as ex: now = datetime.now() print ("Unknown error occured at: " + str(now)) print ("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) now = datetime.now() print ("RDNS Complete: " + str(now)) jobs_manager.record_job_complete() elif args.sonar_file_type == "dns": jobs_manager = JobsManager.JobsManager(mongo_connection, 'get_data_by_cidr_dns') jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.any_url != "": unzipped_dns = download_remote_files(s, html_parser.any_url, save_directory, jobs_manager) update_dns(unzipped_dns, dns_manager, ip_manager, zones) if html_parser.a_url != "": unzipped_dns = download_remote_files(s, html_parser.a_url, save_directory, jobs_manager) update_dns(unzipped_dns, dns_manager, ip_manager, zones) if html_parser.aaaa_url != "": unzipped_dns = download_remote_files(s, html_parser.aaaa_url, save_directory, jobs_manager) update_dns(unzipped_dns, dns_manager, ip_manager, zones) except Exception as ex: now = datetime.now() print ("Unknown error occured at: " + str(now)) print ("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) now = datetime.now() print ("DNS Complete: " + str(now)) jobs_manager.record_job_complete() else: print ("Unrecognized sonar_file_type option. Exiting...") now = datetime.now() print ("Complete: " + str(now))
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Create an instance of the VirusTotal class vt_instance = VirusTotal.VirusTotal() # Get collections for the queries mongo_connector = MongoConnector.MongoConnector() vt_collection = mongo_connector.get_virustotal_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, "get_virustotal_data") jobs_manager.record_job_start() # Collect the list of tracked TLDs zones = ZoneManager.get_distinct_zones(mongo_connector) # For each tracked TLD for zone in zones: logger.debug("Checking " + zone) results = vt_instance.get_domain_report(zone) if results is None: logger.warning("Error querying zone " + zone) elif results["response_code"] == -1: logger.warning("VT unhappy with " + zone) elif results["response_code"] == 0: logger.warning("VT doesn't have " + zone) else: logger.debug("Matched " + zone) results["zone"] = zone results["created"] = datetime.now() # Mongo doesn't allow key names with periods in them # Re-assign to an undotted key name if "Dr.Web category" in results: results["Dr Web category"] = results.pop("Dr.Web category") elif "alphaMountain.ai category" in results: results["alphaMountain_ai category"] = results.pop( "alphaMountain.ai category") vt_collection.delete_one({"zone": zone}) if "last_https_certificate" in results: if "extensions" in results["last_https_certificate"]: if ("1.3.6.1.4.1.11129.2.4.2" in results["last_https_certificate"]["extensions"]): results["last_https_certificate"]["extensions"][ "sct_list"] = results["last_https_certificate"][ "extensions"].pop("1.3.6.1.4.1.11129.2.4.2") mongo_connector.perform_insert(vt_collection, results) # This sleep command is so that we don't exceed the daily limit on the free API # This setting results in this script taking several days to complete time.sleep(25) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Beging Main... """ global global_exit_flag global global_retest_list global global_sleep_time global global_queue_size global global_zgrab_path logger = LoggingUtil.create_log(__name__) global_retest_list = [] parser = argparse.ArgumentParser(description='Launch zgrab against IPs using port 22, 25, 443, or 465.') parser.add_argument('-p', choices=['22', '25', '443', '465'], metavar="port", help='The port to scan: 22, 25, 443, or 465') parser.add_argument('-t', default=5, type=int, metavar="threadCount", help='The number of threads') parser.add_argument('--mx', action="store_true", help='Scan only IPs from MX records. Useful for SMTP scans.') parser.add_argument('-s', default=0, type=int, metavar="sleepTime", help='Sleep time in order to spread out the batches') parser.add_argument('--qs', default=0, type=int, metavar="queueSize", help='How many hosts to scan in a batch') parser.add_argument('--zones_only', action="store_true", help='Scan only IPs from IP zones.') parser.add_argument('--zgrab_path', default=global_zgrab_path, metavar='zgrabVersion', help='The version of ZGrab to use') args = parser.parse_args() if args.p == None: logger.error("A port value (22, 25, 443, or 465) must be provided.") exit(1) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() logger.warning(str(now) + ": I am already running! Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") rm_connector = RemoteMongoConnector.RemoteMongoConnector() all_dns_collection = rm_connector.get_all_dns_connection() ip_manager = IPManager.IPManager(rm_connector, True) jobs_manager = JobsManager.JobsManager(rm_connector, "zgrab_port_ip-" + args.p) jobs_manager.record_job_start() zones_struct = {} zones_struct['zones'] = ZoneManager.get_distinct_zones(rm_connector) # Not pretty but works zones_struct['ip_manager'] = ip_manager if args.mx: (ips, ip_context) = get_mx_ips(zones_struct['zones'], ip_manager, all_dns_collection) elif args.zones_only: (ips, ip_context) = get_only_ipzones(ip_manager.Tracked_CIDRs) else: (ips, ip_context) = get_ips(ip_manager, all_dns_collection) if args.s and int(args.s) > 0: global_sleep_time = int(args.s) if args.qs and int(args.qs) > 0: global_queue_size = int(args.qs) logger.info("Got IPs: " + str(len(ips))) zones_struct['ip_context'] = ip_context zgrab_collection = rm_connector.get_zgrab_port_data_connection() if args.p == "443": run_command = run_port_443_command elif args.p == "22": run_command = run_port_22_command elif args.p == "25": run_command = run_port_25_command elif args.p == "465": run_command = run_port_465_command check_save_location("./json_p" + args.p) global_zgrab_path = args.zgrab_path threads = [] logger.debug("Creating " + str(args.t) + " threads") for thread_id in range (1, args.t + 1): thread = ZgrabThread(thread_id, global_work_queue, args.p, run_command, zones_struct, zgrab_collection) thread.start() threads.append(thread) thread_id += 1 logger.info("Populating Queue") global_queue_lock.acquire() for ip in ips: global_work_queue.put(ip) global_queue_lock.release() # Wait for queue to empty while not global_work_queue.empty(): pass # Notify threads it's time to exit global_exit_flag = 1 # Wait for all threads to complete for t in threads: t.join() logger.info ("Exiting Main Thread") logger.info ("Global retest list: " + str(len(global_retest_list))) # Retest any SMTP hosts that did not respond to the StartTLS handshake if args.p == "25" and len(global_retest_list) > 0: process_thread(logger, global_retest_list, args.p, run_port_25_no_tls_command, zones_struct, zgrab_collection, "retest") # Remove old entries from before the scan if args.p == "443": other_results = zgrab_collection.find({'data.tls': {"$exists": True}, 'data.tls.timestamp': {"$lt": now}}) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.tls': ""}}) elif args.p == "22": if 'zgrab2' in global_zgrab_path: other_results = zgrab_collection.find({'data.ssh': {"$exists": True}, 'data.ssh.timestamp': {"$lt": now}}) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.ssh': ""}}) else: other_results = zgrab_collection.find({'data.xssh': {"$exists": True}, 'data.xssh.timestamp': {"$lt": now}}) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.xssh': ""}}) elif args.p == "25": other_results = zgrab_collection.find({'data.smtp': {"$exists": True}, 'data.smtp.timestamp': {"$lt": now}}) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.smtp': ""}}) elif args.p == "465": other_results = zgrab_collection.find({'data.smtps': {"$exists": True}, 'data.smtps.timestamp': {"$lt": now}}) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.smtps': ""}}) # Remove any completely empty entries zgrab_collection.remove({'data': {}}) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'sonar_round_two') google_dns = GoogleDNS.GoogleDNS() jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) results = dns_manager.find_multiple({'type': 'cname'}, "sonar_dns") round_two = [] round_three = [] # Get all the CNAME values from all_dns and append them to round_two for result in results: if is_tracked_zone(result['value'], zones): round_two.append(result['value']) print("Round two pre-list: " + str(len(round_two))) dead_dns_collection = mongo_connector.get_dead_dns_connection() for value in round_two: is_present = dns_manager.find_count({'fqdn': value}, "sonar_dns") if is_present == 0: print(value + " not found") time.sleep(1) result = google_dns.fetch_DNS_records(value) if result == []: print("Unable to resolve") original_records = dns_manager.find_multiple({"value": value}, "sonar_dns") for record in original_records: check = dead_dns_collection.find({ 'fqdn': record['fqdn'] }).count() if check == 0: record.pop("_id") dead_dns_collection.insert(record) else: for entry in result: if is_tracked_zone(entry['fqdn'], zones): new_record = entry new_record['status'] = 'unconfirmed' new_record['zone'] = get_fld_from_value(value, '') new_record['created'] = datetime.now() if result[0]['type'] == "cname" and is_tracked_zone( entry['value'], zones): add_to_list(entry['value'], round_three) print("Found: " + value) if new_record['zone'] != '': dns_manager.insert_record(new_record, "marinus") # For each tracked CName result found in the first pass across Sonar DNS print("Round Three length: " + str(len(round_three))) for hostname in round_three: zone = get_fld_from_value(hostname, '') if zone != None and zone != '': ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: if is_tracked_zone(ip_addr['fqdn'], zones): record = {"fqdn": ip_addr['fqdn']} record['zone'] = get_fld_from_value( ip_addr['fqdn'], '') record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unconfirmed' dns_manager.insert_record(new_record, "marinus") else: original_record = dns_manager.find_one({"fqdn": hostname}, "marinus") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) print("Failed IP Lookup for: " + hostname) else: print("Failed match on zone for: " + hostname) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now))
def main(): """ Begin main... """ logger = LoggingUtil.create_log(__name__) if is_running("get_censys_files.py"): """ Check to see if a download is in process... """ logger.warning("Can't run due to get_files running. Goodbye!") exit(0) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ logger.warning("I am already running! Goodbye!") exit(0) # Make the relevant database connections RMC = RemoteMongoConnector.RemoteMongoConnector() ip_manager = IPManager.IPManager(RMC) # Verify that the get_files script has a recent file in need of parsing. jobs_collection = RMC.get_jobs_connection() status = jobs_collection.find_one({'job_name': 'censys'}) if status['status'] != "DOWNLOADED": logger.warning("The status is not set to DOWNLOADED. Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Collect the list of available zones zones = ZoneManager.get_distinct_zones(RMC) logger.info("Zones: " + str(len(zones))) # Get the current configuration information for Marinus. config_collection = RMC.get_config_connection() configs = config_collection.find({}) orgs = [] for org in configs[0]['SSL_Orgs']: orgs.append(org) logger.info("Orgs: " + str(len(orgs))) # Obtain the name of the decompressed file. filename_f = open(FILENAME_FILE, "r") decompressed_file = filename_f.readline() filename_f.close() # For manual testing: decompressed_file = "ipv4.json" logger.info("Beginning file processing...") # Remove old results from the database results_collection = RMC.get_results_connection() results_collection.remove({}) all_dns_collection = RMC.get_all_dns_connection() try: with open(decompressed_file, "r") as dec_f: for line in dec_f: try: entry = json.loads(line) """ Does the SSL certificate match a known organization? Is the IP address in a known CIDR? Is the IP address recorded in Splunk? """ if check_in_org(entry, orgs) or \ ip_manager.is_tracked_ip(entry['ip']) or \ ip_manager.find_splunk_data(entry['ip'], "AWS") is not None or \ ip_manager.find_splunk_data(entry['ip'], "AZURE") is not None: entry['zones'] = check_in_zone(entry, zones) entry['aws'] = ip_manager.is_aws_ip(entry['ip']) entry['azure'] = ip_manager.is_azure_ip(entry['ip']) (domains, zones) = lookup_domain(entry, zones, all_dns_collection) if len(domains) > 0: entry['domains'] = domains if len(zones) > 0: for zone in zones: if zone not in entry['zones']: entry['zones'].append(zone) insert_result(entry, results_collection) # else: # #This will add days to the amount of time necessary to scan the file. # matched_zones = check_in_zone(entry, zones) # if matched_zones != []: # entry['zones'] = matched_zones # entry['aws'] = ip_manager.is_aws_ip(entry['ip']) # entry['azure'] = ip_manager.is_azure_ip(entry['ip']) # insert_result(entry, results_collection) except ValueError as err: logger.error("Value Error!") logger.error(str(err)) except: logger.error("Line unexpected error: " + str(sys.exc_info()[0])) logger.error("Line unexpected error: " + str(sys.exc_info()[1])) except IOError as err: logger.error("I/O error({0}): {1}".format(err.errno, err.strerror)) exit(1) except: logger.error("Unexpected error: " + str(sys.exc_info()[0])) logger.error("Unexpected error: " + str(sys.exc_info()[1])) exit(1) # Indicate that the processing of the job is complete and ready for download to Marinus jobs_collection.update_one({'job_name': 'censys'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")