def main(): now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() splunk_query_manager = SplunkQueryManager.SplunkQueryManager() splunk_collection = mongo_connector.get_splunk_connection() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "get_splunk_data") jobs_manager.record_job_start() print("Starting Splunk Query") results_per_page = 100 # Put your custom Splunk search query here. results = splunk_query_manager.do_search('search index=...', results_per_page) print("Processing " + str(splunk_query_manager.RESULTCOUNT) + " results") parse_splunk_results(results, dns_manager, splunk_collection) while True: results = splunk_query_manager.get_next_page() if results is None: break parse_splunk_results(results, dns_manager, splunk_collection) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now))
def create_nodes(graph, mongo_connector, zone, all_certs): """ Create the list of D3.js nodes and links based on the collected certificates """ DNS_MGR = DNSManager.DNSManager(mongo_connector) for cert in all_certs: matched_count = 0 if "censys_count" in cert: matched_count = cert['censys_count'] if "zgrab_count" in cert: matched_count = matched_count + cert['zgrab_count'] graph.add_node(cert['id'], type='certificate', sources=cert['sources'], total_count=matched_count) for dns_entry in cert['dns_entries']: lookup = DNS_MGR.find_one({'fqdn': dns_entry}, None) root_flag = "false" if dns_entry == zone: root_flag = "true" if lookup is None: graph.add_node(dns_entry, root=root_flag, status="No Host", type="domain", sources=cert['sources']) else: graph.add_node(dns_entry, root=root_flag, status="Resolves", type="domain", sources=cert['sources']) graph.add_edge(cert['id'], dns_entry, type="sans") graph.add_edge(dns_entry, cert['id'], type="uses") return graph
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print ("Starting: " + str(now)) logger.info("Starting...") dns_types = {"a":1, "ns":2, "cname":5, "soa":6, "ptr":12, "hinfo": 13, "mx": 15, "txt":16, "aaaa":28, "srv":33, "naptr": 35, "ds": 43, "rrsig": 46, "dnskey": 48} mongo_connector = MongoConnector.MongoConnector() all_dns_collection = mongo_connector.get_all_dns_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, 'marinus_dns') jobs_manager.record_job_start() dns_manager = DNSManager.DNSManager(mongo_connector) zones = ZoneManager.get_distinct_zones(mongo_connector) google_dns = GoogleDNS.GoogleDNS() for zone in zones: time.sleep(1) for dtype, dnum in dns_types.items(): result = google_dns.fetch_DNS_records(zone, dnum) if result == []: logger.debug("No records found for " + zone) else: new_record = result[0] new_record['status'] = 'confirmed' new_record['zone'] = zone new_record['created'] = datetime.now() logger.debug ("Found " + dtype + " for: " + zone) dns_manager.insert_record(new_record, "marinus") logger.info("Starting SOA Search") soa_searches = find_sub_zones(all_dns_collection) for entry in soa_searches: time.sleep(1) result = google_dns.fetch_DNS_records(zone, dns_types['soa']) if result != []: new_record = result[0] new_record['status'] = 'confirmed' new_record['zone'] = get_fld_from_value(entry, '') new_record['created'] = datetime.now() logger.debug ("Found SOA: " + entry) if new_record['zone'] != '': dns_manager.insert_record(new_record, "marinus") jobs_manager.record_job_complete() now = datetime.now() print ("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_collection = mongo_connector.get_jobs_connection() zone_ingestor = ZoneIngestor.ZoneIngestor() current_zones = ZoneManager.get_distinct_zones(mongo_connector) # For cases with multiple R53 accounts, include the account id for reference sts = boto3.client('sts') account_id = sts.get_caller_identity()["Arn"].split(':')[4] r53_source = "R53:" + str(account_id) r53_client = boto3.client('route53') r53_domains = r53_client.list_hosted_zones() r53_zone_list = [] while r53_domains != {}: for zone_data in r53_domains['HostedZones']: # Only add public zones if zone_data['Config']['PrivateZone'] == False: r53_zone_list.append(zone_data) if r53_domains['IsTruncated'] == True: r53_domains = r53_client.list_domains( Marker=r53_domains['NextMarker']) else: r53_domains = {} for zone_data in r53_zone_list: # Double check that this is not a new zone zone_name = zone_data['Name'][:-1] if zone_name not in current_zones: print("Creating zone: " + zone_name) zone_ingestor.add_zone(zone_data['Name'], r53_source) # Add hosts to the zone update_records(r53_client, dns_manager, zone_data, r53_source) # Record status jobs_collection.update_one({'job_name': 'get_route53'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) now = datetime.now() print("Ending: " + str(now))
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zone_ingestor = ZoneIngestor.ZoneIngestor() jobs_manager = JobsManager.JobsManager(mongo_connector, "get_route53") jobs_manager.record_job_start() current_zones = ZoneManager.get_distinct_zones(mongo_connector) # For cases with multiple R53 accounts, include the account id for reference sts = boto3.client("sts") account_id = sts.get_caller_identity()["Arn"].split(":")[4] r53_source = "R53:" + str(account_id) r53_client = boto3.client("route53") r53_domains = r53_client.list_hosted_zones() r53_zone_list = [] while r53_domains != {}: for zone_data in r53_domains["HostedZones"]: # Only add public zones if zone_data["Config"]["PrivateZone"] == False: r53_zone_list.append(zone_data) if r53_domains["IsTruncated"] == True: r53_domains = r53_client.list_domains( Marker=r53_domains["NextMarker"]) else: r53_domains = {} for zone_data in r53_zone_list: # Double check that this is not a new zone zone_name = zone_data["Name"][:-1] if zone_name not in current_zones: logger.info("Creating zone: " + zone_name) zone_ingestor.add_zone(zone_data["Name"], r53_source) # Add hosts to the zone update_records(r53_client, dns_manager, zone_data, r53_source) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "get_external_cnames") jobs_manager.record_job_start() groups = {} # Collect zones zone_results = ZoneManager.get_distinct_zones(mongo_connector) zones = [] for zone in zone_results: if zone.find(".") >= 0: zones.append(zone) # Collect the all_dns cnames. logger.info("Starting All DNS...") all_dns_recs = dns_manager.find_multiple({"type": "cname"}, None) for srec in all_dns_recs: if not is_tracked_zone(srec["value"], zones): add_to_list( get_fld_from_value(srec["value"], srec["zone"]), srec["fqdn"], srec["value"], srec["zone"], groups, ) # Update the database tpds_collection = mongo_connector.get_tpds_connection() tpds_collection.delete_many({}) for key in groups.keys(): tpds_collection.insert_one(groups[key]) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): now = datetime.now() print ("Starting: " + str(now)) dns_types = {"a":1, "ns":2, "cname":5, "soa":6, "ptr":12, "hinfo": 13, "mx": 15, "txt":16, "aaaa":28, "srv":33, "naptr": 35, "ds": 43, "rrsig": 46, "dnskey": 48} mongo_connector = MongoConnector.MongoConnector() all_dns_collection = mongo_connector.get_all_dns_connection() jobs_collection = mongo_connector.get_jobs_connection() dns_manager = DNSManager.DNSManager(mongo_connector) zones = ZoneManager.get_distinct_zones(mongo_connector) google_dns = GoogleDNS.GoogleDNS() for zone in zones: time.sleep(1) for dtype, dnum in dns_types.items(): result = google_dns.fetch_DNS_records(zone, dnum) if result == []: print("No records found for " + zone) else: new_record = result[0] new_record['status'] = 'confirmed' new_record['zone'] = zone new_record['created'] = datetime.now() print ("Found " + dtype + " for: " + zone) dns_manager.insert_record(new_record, "marinus") print("Starting SOA Search") soa_searches = find_sub_zones(all_dns_collection) for entry in soa_searches: time.sleep(1) result = google_dns.fetch_DNS_records(zone, dns_types['soa']) if result != []: new_record = result[0] new_record['status'] = 'confirmed' new_record['zone'] = get_fld_from_value(entry, '') new_record['created'] = datetime.now() print ("Found SOA: " + entry) if new_record['zone'] != '': dns_manager.insert_record(new_record, "marinus") jobs_collection.update_one({'job_name': 'marinus_dns'}, {'$currentDate': {"updated" : True}, "$set": {'status': 'COMPLETE'}}) now = datetime.now() print ("Complete: " + str(now))
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_collection = mongo_connector.get_jobs_connection() groups = {} # Collect zones zone_results = ZoneManager.get_distinct_zones(mongo_connector) zones = [] for zone in zone_results: if zone.find(".") >= 0: zones.append(zone) # Collect the all_dns cnames. print("Starting All DNS...") all_dns_recs = dns_manager.find_multiple({'type': 'cname'}, None) for srec in all_dns_recs: if not is_tracked_zone(srec['value'], zones): add_to_list(get_fld_from_value(srec['value'], srec['zone']), srec['fqdn'], srec['value'], srec['zone'], groups) # Update the database tpds_collection = mongo_connector.get_tpds_connection() tpds_collection.remove({}) for key in groups.keys(): tpds_collection.insert(groups[key]) # Record status jobs_collection.update_one({'job_name': 'get_external_cnames'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) now = datetime.now() print("Ending: " + str(now))
def find_dns_zones(self, ip): """ Find DNS zones related to the IP address. """ dns_manager = DNSManager.DNSManager(self.mongo_connector) results = dns_manager.find_multiple({"value": ip}, None) zones = [] domains = [] for result in results: if result['zone'] not in zones and result['zone'] != '': zones.append(result['zone']) if result['fqdn'] not in domains and result['fqdn'] != ip: domains.append(result['fqdn']) return zones, domains
def add_new_domain_names(hostnames, zones, mongo_connector): """ Perform a GoogleDNS lookup on all identified domain names and add them to the DNS tracker. """ google_dns = GoogleDNS.GoogleDNS() dns_manager = DNSManager.DNSManager(mongo_connector) for hostname in hostnames: results = google_dns.fetch_DNS_records(hostname) if results != []: for result in results: temp_zone = get_tracked_zone(result['fqdn'], zones) if temp_zone is not None: new_record = {"fqdn": result['fqdn']} new_record['zone'] = temp_zone new_record['created'] = datetime.now() new_record['type'] = result['type'] new_record['value'] = result['value'] new_record['status'] = 'unknown' dns_manager.insert_record(new_record, "ssl")
def main(): """ Begin Main """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Set up all the database connections mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'create_netaddr_graphs') jobs_manager.record_job_start() # Get the list of the all Class C's in Marinus groups = [] create_list_of_cidrs(groups, mongo_connector, dns_manager) # Create a separate copy of the class C list since groups will be modified later cidr_list = groups + [] # Create the stats on the network data group_data = create_network_data_sets(groups, mongo_connector) logger.info("Number of Tracked Class C's: " + str(group_data['tracked_count'])) logger.info("Number of AWS Class C's: " + str(group_data['aws_count'])) logger.info("Number of Azure Class C's: " + str(group_data['azure_count'])) logger.info("Number of Akamai Class C's: " + str(group_data['akamai_count'])) logger.info("Number of Class C's: " + str(len(groups))) # Get the current list of zones zones = ZoneManager.get_distinct_zones(mongo_connector) # For each Class C that was identified in Marinus... for tcidr in cidr_list: cidr = tcidr.replace(REPLACE_CHAR, ".") groups = [] graph = nx.Graph() add_to_list(cidr, groups) graph.add_node(cidr, data_type="class_c", type=0, depends=[], dependedOnBy=[], docs="<h1>Parent</h1>") find_all_dns_by_zone(graph, cidr, groups, dns_manager) find_srdns_by_zone(graph, cidr, groups, mongo_connector) data = json_graph.node_link_data(graph) reformat_data(data, cidr, groups) new_data = {} new_data['directed'] = data['directed'] new_data['graph'] = data['graph'] new_data['multigraph'] = data['multigraph'] new_data['errs'] = [] new_data['links'] = data['links'] new_data['data'] = {} for i in range(0, len(data['nodes'])): new_data['data'][data['nodes'][i]['id'].replace( ".", REPLACE_CHAR)] = data['nodes'][i] config = {} config['title'] = cidr + " Network Map" config['graph'] = {} config['graph']['linkDistance'] = 150 config['graph']['charge'] = -400 config['graph']['height'] = 800 config['graph']['numColors'] = len(groups) config['graph']['labelPadding'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['labelMargin'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['ticksWithoutCollisions'] = 50 config['graph_type'] = "cidr" config['types'] = {} regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$" regx = re.compile(regex_str) for tgroup in groups: group = tgroup.replace(REPLACE_CHAR, ".") data_type = "tpd" if group in zones: data_type = "tracked_domain" elif re.match(regx, group): data_type = "cidr" config['types'][tgroup] = { "short": group, "long": "A group from the network: " + group, "data_type": data_type } config['constraints'] = [] tmp = int(math.ceil(math.sqrt(len(groups)))) + 1 x = [] y = [] for i in range(1, tmp): val = round((i * 1.0) / tmp, 2) x.append(str(val)) y.append(str(val)) x_pos = 0 y_pos = 0 for group in groups: config['constraints'].append({ "has": { "type": group }, "type": "position", "x": x[x_pos], "y": y[y_pos] }) x_pos = x_pos + 1 if x_pos >= len(x): x_pos = 0 y_pos = y_pos + 1 config['jsonUrl'] = "/api/v1.0/cidr_graphs/" + cidr new_data['config'] = config new_data['created'] = datetime.now() new_data['zone'] = cidr cidr_graphs_collection = mongo_connector.get_cidr_graphs_connection() cidr_graphs_collection.remove({'zone': cidr}) cidr_graphs_collection.insert_one(new_data) time.sleep(1) # Remove last week's old entries lastweek = datetime.now() - timedelta(days=7) cidr_graphs_collection.remove({'created': {"$lt": lastweek}}) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() all_dns_collection = mongo_connector.get_all_dns_connection() dns_manager = DNSManager.DNSManager(mongo_connector) GDNS = GoogleDNS.GoogleDNS() ip_manager = IPManager.IPManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'remove_expired_entries') jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) # The sources for which to remove expired entries results = mongo_connector.perform_distinct(all_dns_collection, 'sources.source') sources = [] for source in results: temp = {} temp['name'] = source if "common_crawl" in source: temp['diff'] = -4 else: temp['diff'] = -2 sources.append(temp) # Before completely removing old entries, make an attempt to see if they are still valid. # Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source. # Rather than throw away valid information, this will archive it. for entry in sources: removal_date = monthdelta(datetime.now(), entry['diff']) source = entry['name'] logger.debug("Removing " + source + " as of: " + str(removal_date)) last_domain = "" results = all_dns_collection.find({ 'sources': { "$size": 1 }, 'sources.source': source, 'sources.updated': { "$lt": removal_date } }) for result in results: if result['fqdn'] != last_domain: last_domain = result['fqdn'] lookup_int = get_lookup_int(logger, result, GDNS) dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int) if dns_result != []: insert_current_results(dns_result, dns_manager, zones, result, source) dns_manager.remove_all_by_source_and_date(source, entry['diff']) # Get the date for today minus two months d_minus_2m = monthdelta(datetime.now(), -2) logger.info("Removing SRDNS as of: " + str(d_minus_2m)) # Remove the old records srdns_collection = mongo_connector.get_sonar_reverse_dns_connection() srdns_collection.remove({'updated': {"$lt": d_minus_2m}}) ip_manager.delete_records_by_date(d_minus_2m) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete")
class UltraDNSZonesInfo(object): UH = UltraDNSHelper.UltraDNSHelper('get_ultradns_zones_info') APIH = APIHelper.APIHelper() DNS_MGR = DNSManager.DNSManager(UH.MC) def __ultradns_zone_info_response_handler(self, response): """ Handles the API response. Incorrect JSON parsing is allowed upto 20 times post which the script exits. :param response: Response object """ try: response_data = response.json() record_sets = response_data['rrSets'] except (ValueError, AttributeError) as err: if self.UH.incorrect_response_json_allowed > 0: print('Unable to parse response JSON for zone ' + self.zone_queried) self.UH.incorrect_response_json_allowed -= 1 else: self.APIH.handle_api_error( 'Unable to parse response JSON for 20 zones: ' + repr(err), self.UH.jobs_manager, ) else: for record in record_sets: dns_info = dict() # The ownerName could be either the FQDN or a relative domain name. # In case it is a FQDN it will end in '.' fqdn = record['ownerName'] + '.' + self.zone_queried if record['ownerName'].endswith('.'): fqdn = record['ownerName'][:-1] dns_info['zone'] = self.zone_queried dns_info['fqdn'] = fqdn dns_info['type'] = record['rrtype'].split(' ')[0].lower() dns_info['status'] = 'unknown' for dns in record['rdata']: if dns_info['type'] in ['a', 'ptr']: try: if IPAddress(dns).is_private(): continue except AddrFormatError as err: print('For ' + fqdn + ' encountered: ' + str(err)) continue if not (dns_info['type'] in ['soa', 'txt' ]) and dns.endswith('.'): dns = dns[:-1] dns_info['value'] = dns dns_info['created'] = datetime.now() self.DNS_MGR.insert_record(dns_info.copy(), self.UH.source) self.UH.set_offset(response_data['resultInfo']) def __paginated_ultradns_zones_info_request(self): """ Makes paginated API calls to UltraDNS. The API is retried 3 times in case of ConnectionError exception before the script exists. The script exists on encountering HTTPError or any other RequestException. In case a 401 is encountered along with the required token expiration message, another login API is sent with grant_type set as 'refresh_token' to retrieve a valid access token. """ url = self.UH.ULTRACONNECT.ZONEINFO.format( zone_queried=self.zone_queried) try: response = self.UH.backoff_api_retry( url, { 'q': 'kind:RECORDS', 'limit': 2000, 'offset': self.UH.offset, }, {'authorization': 'Bearer ' + self.UH.access_token}) response.raise_for_status() except requests.exceptions.HTTPError as herr: err_msg = json.loads(response.text)['errorMessage'] if response.status_code == 401 and err_msg == self.UH.access_token_expiration_error: self.UH.login('refresh_token') self.__paginated_ultradns_zones_info_request() else: self.APIH.handle_api_error(herr, self.UH.jobs_manager) except requests.exceptions.RequestException as err: self.APIH.handle_api_error(err, self.UH.jobs_manager) else: self.__ultradns_zone_info_response_handler(response) def __get_ultradns_zones_info(self): """ Extracts the zone DNS information from UltraDNS in a paginated manner for the UltraDNS zones. """ print("Starting: " + str(datetime.now())) self.UH.jobs_manager.record_job_start() self.UH.get_previous_zones() # For querying UltraDNS, we need to query on the exact zones reported # hence we query for previous_zones. for zone in self.UH.previous_zones: self.zone_queried = zone self.UH.offset = 0 self.__paginated_ultradns_zones_info_request() while self.UH.offset: self.__paginated_ultradns_zones_info_request() # Record status self.UH.jobs_manager.record_job_complete() print("Ending: " + str(datetime.now())) def __init__(self): self.__get_ultradns_zones_info()
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) if is_running(os.path.basename(__file__)): logger.warning("Already running...") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") r7 = Rapid7.Rapid7() mongo_connection = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connection) ip_manager = IPManager.IPManager(mongo_connection) rdns_collection = mongo_connection.get_sonar_reverse_dns_connection() zones = ZoneManager.get_distinct_zones(mongo_connection) logger.info("Zone length: " + str(len(zones))) save_directory = "./files/" parser = argparse.ArgumentParser( description='Parse Sonar files based on CIDRs.') parser.add_argument('--sonar_file_type', required=True, help='Specify "dns" or "rdns"') args = parser.parse_args() check_save_location(save_directory) # A session is necessary for the multi-step log-in process s = requests.Session() if args.sonar_file_type == "rdns": jobs_manager = JobsManager.JobsManager(mongo_connection, 'get_data_by_cidr_rdns') jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "rdns", jobs_manager) if html_parser.rdns_url == "": logger.error("Unknown Error") jobs_manager.record_job_error() exit(0) unzipped_rdns = download_remote_files(logger, s, html_parser.rdns_url, save_directory, jobs_manager) update_rdns(logger, unzipped_rdns, rdns_collection, dns_manager, ip_manager, zones) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) logger.info("RDNS Complete") jobs_manager.record_job_complete() elif args.sonar_file_type == "dns": jobs_manager = JobsManager.JobsManager(mongo_connection, 'get_data_by_cidr_dns') jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.any_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.any_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, dns_manager, ip_manager, zones) if html_parser.a_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.a_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, dns_manager, ip_manager, zones) if html_parser.aaaa_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.aaaa_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, dns_manager, ip_manager, zones) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) logger.info("DNS Complete") jobs_manager.record_job_complete() else: logger.error("Unrecognized sonar_file_type option. Exiting...") now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): now = datetime.now() print("Starting: " + str(now)) azure_connector = AzureConnector.AzureConnector() mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zone_ingestor = ZoneIngestor.ZoneIngestor() jobs_manager = JobsManager.JobsManager(mongo_connector, 'fetch_azure_dns') jobs_manager.record_job_start() current_zones = ZoneManager.get_distinct_zones(mongo_connector) resource_client = azure_connector.get_resources_client() resources = [] # The resource list is not currently used. for item in resource_client.resource_groups.list(): resources.append(item.name) dns_client = azure_connector.get_dns_client() zones = dns_client.zones.list() # The type of records the Azure DNS will let you configure record_types = { 'A': 'arecords', 'AAAA': 'aaaa_records', 'MX': 'mx_records', 'NS': 'ns_records', 'PTR': 'ptr_records', 'SRV': 'srv_records', 'TXT': 'txt_records', 'CNAME': 'cname_record', 'SOA': 'soa_record' } for zone in zones: print("Zone: " + zone.name) data = split_id(zone.id) if zone.zone_type == ZoneType.public: print(zone.name + " is public:") if zone.name not in current_zones: print("Creating zone: " + zone.name) zone_ingestor.add_zone(zone.name, "azure:" + data["resourceGroups"]) try: print("ResourceGroup: " + data["resourceGroups"]) records = dns_client.record_sets.list_all_by_dns_zone( data["resourceGroups"], zone.name) for entry in records: # The record_data id value ends in rtype/rvalue so you must guess the rtype record_data = split_id(entry.id) for rtype in record_types: if rtype in record_data: results = extract_record_set_value(rtype, entry) for result in results: result['zone'] = zone.name result['created'] = datetime.now() result['status'] = 'confirmed' dns_manager.insert_record( result, "azure:" + data["resourceGroups"]) except: print("No records found") jobs_manager.record_job_complete()
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) if is_running(os.path.basename(__file__)): logger.warning("Already running...") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") parser = argparse.ArgumentParser( description="Parse Sonar files based on domain zones.") parser.add_argument( "--sonar_file_type", choices=["dns-any", "dns-a", "rdns"], required=True, help='Specify "dns-any", "dns-a", or "rdns"', ) parser.add_argument( "--database", choices=["local", "remote"], required=False, default="local", help="Whether to use the local or remote DB", ) args = parser.parse_args() r7 = Rapid7.Rapid7() if args.database == "remote": mongo_connector = RemoteMongoConnector.RemoteMongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector, "get_sonar_data_dns") else: mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zones = ZoneManager.get_distinct_zones(mongo_connector) save_directory = "./files/" check_save_location(save_directory) # A session is necessary for the multi-step log-in process s = requests.Session() if args.sonar_file_type == "rdns": logger.info("Updating RDNS records") jobs_manager = JobsManager.JobsManager(mongo_connector, "get_sonar_data_rdns") jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "rdns", jobs_manager) if html_parser.rdns_url == "": logger.error("Unknown Error") jobs_manager.record_job_error() exit(0) unzipped_rdns = download_remote_files(logger, s, html_parser.rdns_url, save_directory, jobs_manager) update_rdns(logger, unzipped_rdns, zones, dns_manager, mongo_connector) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) jobs_manager.record_job_complete() elif args.sonar_file_type == "dns-any": logger.info("Updating DNS ANY records") jobs_manager = JobsManager.JobsManager(mongo_connector, "get_sonar_data_dns-any") jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.any_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.any_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, dns_manager) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) jobs_manager.record_job_complete() elif args.sonar_file_type == "dns-a": logger.info("Updating DNS A, AAAA, and CNAME records") jobs_manager = JobsManager.JobsManager(mongo_connector, "get_sonar_data_dns-a") jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.a_url != "": logger.info("Updating A records") unzipped_dns = download_remote_files(logger, s, html_parser.a_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, dns_manager) if html_parser.aaaa_url != "": logger.info("Updating AAAA records") unzipped_dns = download_remote_files(logger, s, html_parser.aaaa_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, dns_manager) if html_parser.cname_url != "": logger.info("Updating CNAME records") unzipped_dns = download_remote_files(logger, s, html_parser.cname_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, dns_manager) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) jobs_manager.record_job_complete() else: logger.error("Unrecognized sonar_file_type option. Exiting...") now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): mongo_connector = MongoConnector.MongoConnector() now = datetime.now() print("Starting: " + str(now)) jobs_manager = JobsManager.JobsManager(mongo_connector, 'owasp_amass') zones = ZoneManager.get_distinct_zones(mongo_connector) dns_manager = DNSManager.DNSManager(mongo_connector) output_dir = "./amass_files/" arg_parser = argparse.ArgumentParser( description= 'Run the OWASP Amass tool and store the results in the database.') arg_parser.add_argument( '--config_file', required=False, help='An optional Amass config file. Otherwise, defaults will be used.' ) arg_parser.add_argument('--amass_path', required=True, help='The path to the amass binary') arg_parser.add_argument('--output_dir', default=output_dir, help="The path where to save Amass files.") arg_parser.add_argument('--amass_version', type=int, default=3, help='The version of OWASP Amass being used.') arg_parser.add_argument( '--sleep', type=int, default=5, help= 'Sleep time in seconds between amass runs so as not to overuse service limits.' ) args = arg_parser.parse_args() if not os.path.isfile(args.amass_path): print("ERROR: Incorrect amass_path argument provided") exit(1) if 'config_file' in args and not os.path.isfile(args.config_file): print("ERROR: Incorrect config_file location") exit(1) if 'output_dir' in args: output_dir = args.output_dir if not output_dir.endswith("/"): output_dir = output_dir + "/" check_save_location(output_dir) jobs_manager.record_job_start() for zone in zones: # Pace out calls to the Amass services time.sleep(args.sleep) command_line = [] command_line.append(args.amass_path) if int(args.amass_version) >= 3: command_line.append("enum") if args.config_file: command_line.append("-config") command_line.append(args.config_file) command_line.append("-d") command_line.append(zone) command_line.append("-src") command_line.append("-ip") command_line.append("-do") command_line.append(output_dir + zone + "-do.json") try: subprocess.check_call(command_line) except subprocess.CalledProcessError as e: # Even when there is an error, there will likely still be results. # We can continue with the data that was collected thus far. print("ERROR: Amass run exited with a non-zero status: " + str(e)) if os.path.isfile(output_dir + zone + "-do.json"): output = open(output_dir + zone + "-do.json", "r") json_data = [] for line in output: try: json_data.append(json.loads(line)) except: print("Amass wrote an incomplete line: " + str(line)) output.close() for finding in json_data: if 'type' in finding and finding[ 'type'] == 'infrastructure' or finding[ 'type'] == 'domain': # Not currently recording continue elif is_tracked_zone(finding['domain'], zones): record_finding(dns_manager, finding) else: # print("Skipping: " + finding['domain'] + " type: " + finding['type']) pass jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now))
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "sonar_round_two") google_dns = GoogleDNS.GoogleDNS() jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) results = dns_manager.find_multiple({"type": "cname"}, "sonar_dns") round_two = [] round_three = [] # Get all the CNAME values from all_dns and append them to round_two for result in results: if is_tracked_zone(result["value"], zones): round_two.append(result["value"]) logger.info("Round two pre-list: " + str(len(round_two))) dead_dns_collection = mongo_connector.get_dead_dns_connection() for value in round_two: is_present = dns_manager.find_count({"fqdn": value}, "sonar_dns") if is_present == 0: logger.debug(value + " not found") time.sleep(1) result = google_dns.fetch_DNS_records(value) if result == []: logger.debug("Unable to resolve") original_records = dns_manager.find_multiple({"value": value}, "sonar_dns") for record in original_records: check = dead_dns_collection.count_documents( {"fqdn": record["fqdn"]}) if check == 0: record.pop("_id") dead_dns_collection.insert(record) else: for entry in result: if is_tracked_zone(entry["fqdn"], zones): new_record = entry new_record["status"] = "unconfirmed" new_record["zone"] = get_fld_from_value(value, "") new_record["created"] = datetime.now() if result[0]["type"] == "cname" and is_tracked_zone( entry["value"], zones): add_to_list(entry["value"], round_three) logger.debug("Found: " + value) if new_record["zone"] != "": dns_manager.insert_record(new_record, "marinus") # For each tracked CName result found in the first pass across Sonar DNS logger.info("Round Three length: " + str(len(round_three))) for hostname in round_three: zone = get_fld_from_value(hostname, "") if zone != None and zone != "": ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: if is_tracked_zone(ip_addr["fqdn"], zones): record = {"fqdn": ip_addr["fqdn"]} record["zone"] = get_fld_from_value( ip_addr["fqdn"], "") record["created"] = datetime.now() record["type"] = ip_addr["type"] record["value"] = ip_addr["value"] record["status"] = "unconfirmed" dns_manager.insert_record(new_record, "marinus") else: original_record = dns_manager.find_one({"fqdn": hostname}, "marinus") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) logger.debug("Failed IP Lookup for: " + hostname) else: logger.debug("Failed match on zone for: " + hostname) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
class UltraDNSZonesInfo(object): UH = UltraDNSHelper.UltraDNSHelper("get_ultradns_zones_info") APIH = APIHelper.APIHelper() DNS_MGR = DNSManager.DNSManager(UH.MC) _logger = None def __ultradns_zone_info_response_handler(self, response): """ Handles the API response. Incorrect JSON parsing is allowed upto 20 times post which the script exits. :param response: Response object """ try: response_data = response.json() record_sets = response_data["rrSets"] except (ValueError, AttributeError) as err: if self.UH.incorrect_response_json_allowed > 0: self._logger.warning( "Unable to parse response JSON for zone " + self.zone_queried) self.UH.incorrect_response_json_allowed -= 1 else: self.APIH.handle_api_error( "Unable to parse response JSON for 20 zones: " + repr(err), self.UH.jobs_manager, ) else: for record in record_sets: dns_info = dict() # The ownerName could be either the FQDN or a relative domain name. # In case it is a FQDN it will end in '.' fqdn = record["ownerName"] + "." + self.zone_queried if record["ownerName"].endswith("."): fqdn = record["ownerName"][:-1] # A get_root_domain lookup is performed because UDNS supports sub-zones dns_info["zone"] = ZoneManager.get_root_domain( self.zone_queried) dns_info["fqdn"] = fqdn dns_info["type"] = record["rrtype"].split(" ")[0].lower() dns_info["status"] = "unknown" for dns in record["rdata"]: if dns_info["type"] in ["a", "ptr"]: try: if IPAddress(dns).is_private(): continue except AddrFormatError as err: self._logger.warning("For " + fqdn + " encountered: " + str(err)) continue if not (dns_info["type"] in ["soa", "txt" ]) and dns.endswith("."): dns = dns[:-1] dns_info["value"] = dns dns_info["created"] = datetime.now() self.DNS_MGR.insert_record(dns_info.copy(), self.UH.source) self.UH.set_offset(response_data["resultInfo"]) def __paginated_ultradns_zones_info_request(self): """ Makes paginated API calls to UltraDNS. The API is retried 3 times in case of ConnectionError exception before the script exists. The script exists on encountering HTTPError or any other RequestException. In case a 401 is encountered along with the required token expiration message, another login API is sent with grant_type set as 'refresh_token' to retrieve a valid access token. """ url = self.UH.ULTRACONNECT.ZONEINFO.format( zone_queried=self.zone_queried) try: response = self.UH.backoff_api_retry( url, { "q": "kind:RECORDS", "limit": 2000, "offset": self.UH.offset, }, {"authorization": "Bearer " + self.UH.access_token}, ) response.raise_for_status() except requests.exceptions.HTTPError as herr: message = json.loads(response.text) if isinstance(message, list): err_msg = json.loads(response.text)[0]["errorMessage"] else: err_msg = json.loads(response.text)["errorMessage"] if (response.status_code == 401 and err_msg == self.UH.access_token_expiration_error): self.UH.login("refresh_token") self.__paginated_ultradns_zones_info_request() elif response.status_code == 404: self._logger.warning("ERROR: Could not find data for: " + str(self.zone_queried)) else: self.APIH.handle_api_error(herr, self.UH.jobs_manager) except requests.exceptions.RequestException as err: self.APIH.handle_api_error(err, self.UH.jobs_manager) else: self.__ultradns_zone_info_response_handler(response) def __get_ultradns_zones_info(self): """ Extracts the zone DNS information from UltraDNS in a paginated manner for the UltraDNS zones. """ print("Starting: " + str(datetime.now())) self._logger.info("Starting...") self.UH.jobs_manager.record_job_start() self.UH.get_previous_zones() # For querying UltraDNS, we need to query on the exact zones reported # hence we query for previous_zones. for zone in self.UH.previous_zones: self.zone_queried = zone self.UH.offset = 0 self.__paginated_ultradns_zones_info_request() while self.UH.offset: self.__paginated_ultradns_zones_info_request() # Record status self.UH.jobs_manager.record_job_complete() print("Ending: " + str(datetime.now())) self._logger.info("Complete.") def __init__(self): self._logger = LoggingUtil.create_log(__name__) self.__get_ultradns_zones_info()
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'extract_mx_domains') google_dns = GoogleDNS.GoogleDNS() jobs_manager.record_job_start() dns_names = [] round_two = [] zones = ZoneManager.get_distinct_zones(mongo_connector) # Collect the list of domains from the MX Records extract_mx_names(dns_names, dns_manager) input_list = [] # Some MX records point to the third-party domains. # Therefore, we filter to only the root domains that belong to the tracked company. print("Pre-filter list: " + str(len(dns_names))) for hostname in dns_names: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) # Pause to prevent DoS-ing of Google's HTTPS DNS Service time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) if ip_addr['type'] == "cname" and is_tracked_zone( ip_addr['value'], zones): add_to_round_two(ip_addr['value'], round_two) else: print("Failed IP Lookup for: " + hostname) else: print("Failed match on zone for: " + hostname) dead_dns_collection = mongo_connector.get_dead_dns_connection() # Some DNS records will be CNAME records pointing to other tracked domains. # This is a single level recursion to lookup those domains. print("Round Two list: " + str(len(round_two))) for hostname in round_two: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) else: print("Failed IP Lookup for: " + hostname) original_record = dns_manager.find_one({"fqdn": hostname}, "mx") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) else: print("Failed match on zone for: " + hostname) # Record all the results. dns_manager.remove_by_source("mx") print("List length: " + str(len(input_list))) for final_result in input_list: dns_manager.insert_record(final_result, "mx") # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now))
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Connect to the remote databases mongo_connector = MongoConnector.MongoConnector() rm_connector = RemoteMongoConnector.RemoteMongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zones = ZoneManager.get_distinct_zones(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "remote_download") jobs_manager.record_job_start() remote_jobs_collection = rm_connector.get_jobs_connection() # Check the status of the Censys job on the remote database try: status = remote_jobs_collection.find_one({"job_name": "censys"}) except: logger.error("Can not connect to remote database") jobs_manager.record_job_error() exit(1) if (status is not None and "status" in status and status["status"] != jobs_manager.COMPLETE): logger.info("Censys scans status is not COMPLETE") elif (status is not None and "status" in status and status["status"] == jobs_manager.COMPLETE): # Get connections to the relevant collections. censys_collection = mongo_connector.get_zgrab_443_data_connection() remote_censys_collection = rm_connector.get_zgrab_443_data_connection() download_censys_scan_info(logger, censys_collection, remote_censys_collection) # Tell the remote database that is safe to start processing the next Censys file remote_jobs_collection.update_one( {"job_name": "censys"}, { "$currentDate": { "updated": True }, "$set": { "status": jobs_manager.READY } }, ) # Get connections to the relevant HTTPS collections. zgrab_443_data_collection = mongo_connector.get_zgrab_443_data_connection() remote_zgrab_443_data_collection = rm_connector.get_zgrab_443_data_connection( ) download_zgrab_info(logger, zgrab_443_data_collection, remote_zgrab_443_data_collection) # Get connections to the relevant HTTP collections. zgrab_80_data_collection = mongo_connector.get_zgrab_80_data_connection() remote_zgrab_80_data_collection = rm_connector.get_zgrab_80_data_connection( ) download_zgrab_info(logger, zgrab_80_data_collection, remote_zgrab_80_data_collection) # Get connections to the relevant port collections. zgrab_port_data_collection = mongo_connector.get_zgrab_port_data_connection( ) remote_zgrab_port_data_collection = rm_connector.get_zgrab_port_data_connection( ) download_zgrab_port_info(logger, zgrab_port_data_collection, remote_zgrab_port_data_collection) # Download latest whois information status = remote_jobs_collection.find_one({"job_name": "whois_lookups"}) if status["status"] == jobs_manager.COMPLETE: whois_collection = mongo_connector.get_whois_connection() remote_whois_collection = rm_connector.get_whois_connection() download_whois_data(logger, whois_collection, remote_whois_collection) remote_jobs_collection.update_one( {"job_name": "whois"}, {"$set": { "status": jobs_manager.READY }}) # Download Amass results amass_collection = mongo_connector.get_owasp_amass_connection() remote_amass_collection = rm_connector.get_owasp_amass_connection() download_amass_data(logger, amass_collection, remote_amass_collection, dns_manager, zones) # Download the status of the remote jobs download_jobs_status(logger, jobs_manager._jobs_collection, remote_jobs_collection) # Download remote sonar DNS findings download_sonar_dns(logger, dns_manager, rm_connector) # Download remote sonar RDNS findings download_sonar_rdns(logger, mongo_connector, rm_connector) # Update the local jobs database to done jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) graphs_collection = mongo_connector.get_graphs_connection() graphs_data_collection = mongo_connector.get_graphs_data_connection() graphs_links_collection = mongo_connector.get_graphs_links_connection() graphs_docs_collection = mongo_connector.get_graphs_docs_connection() ip_manager = IPManager.IPManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'create_graphs2') jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) for zone in zones: groups = [] graph = nx.Graph() add_to_list(zone, groups) graph.add_node(zone, data_type="tld", type=0, depends=[], dependedOnBy=[], docs="<h1>Parent</h1>") find_all_dns_by_zone(graph, zone, groups, dns_manager, ip_manager) find_srdns_by_zone(graph, zone, groups, mongo_connector, ip_manager) data = json_graph.node_link_data(graph) reformat_data(data, zone, groups) new_data = {} new_data['directed'] = data['directed'] new_data['graph'] = data['graph'] new_data['multigraph'] = data['multigraph'] new_data['errs'] = [] config = {} config['title'] = zone + " Network Map" config['graph'] = {} config['graph']['linkDistance'] = 150 config['graph']['charge'] = -400 config['graph']['height'] = 800 config['graph']['numColors'] = len(groups) config['graph']['labelPadding'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['labelMargin'] = { "left": 3, "right": 3, "top": 2, "bottom": 2 } config['graph']['ticksWithoutCollisions'] = 50 config['graph_type'] = "tracked_domain" config['types'] = {} regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$" regx = re.compile(regex_str) for tgroup in groups: group = tgroup.replace(REPLACE_CHAR, ".") data_type = "tpd" if group in zones: data_type = "tracked_domain" elif re.match(regx, group): data_type = "cidr" config['types'][tgroup] = { "short": group, "long": "A group from the network: " + group, "data_type": data_type } config['constraints'] = [] tmp = int(math.ceil(math.sqrt(len(groups)))) + 1 x = [] y = [] for i in range(1, tmp): val = round((i * 1.0) / tmp, 2) x.append(str(val)) y.append(str(val)) x_pos = 0 y_pos = 0 for group in groups: config['constraints'].append({ "has": { "type": group }, "type": "position", "x": x[x_pos], "y": y[y_pos] }) x_pos = x_pos + 1 if x_pos >= len(x): x_pos = 0 y_pos = y_pos + 1 config['jsonUrl'] = "/api/v1.0/graphs/" + zone new_data['config'] = config new_data['created'] = datetime.now() new_data['zone'] = zone new_docs_data = {} new_docs_data['docs'] = {} new_docs_data['zone'] = zone new_docs_data['created'] = datetime.now() new_graph_data = {} new_graph_data['data'] = {} for i in range(0, len(data['nodes'])): new_graph_data['data'][data['nodes'][i]['id'].replace( ".", REPLACE_CHAR)] = data['nodes'][i] new_docs_data['docs'][data['nodes'][i]['id'].replace( ".", REPLACE_CHAR)] = data['nodes'][i]['docs'] del new_graph_data['data'][data['nodes'][i]['id'].replace( ".", REPLACE_CHAR)]['docs'] new_graph_data['created'] = datetime.now() new_graph_data['zone'] = zone new_graph_data['directed'] = data['directed'] new_graph_data['multigraph'] = data['multigraph'] new_graph_data['errs'] = [] new_links_data = {} new_links_data['links'] = data['links'] new_links_data['created'] = datetime.now() new_links_data['zone'] = zone new_links_data['directed'] = data['directed'] new_links_data['multigraph'] = data['multigraph'] new_links_data['errs'] = [] try: graphs_collection.remove({'zone': zone}) graphs_collection.insert_one(new_data) graphs_data_collection.remove({'zone': zone}) graphs_data_collection.insert_one(new_graph_data) graphs_links_collection.remove({'zone': zone}) graphs_links_collection.insert_one(new_links_data) graphs_docs_collection.remove({'zone': zone}) graphs_docs_collection.insert_one(new_docs_data) except: logger.error("ERROR: Can't insert: " + zone) time.sleep(1) # Remove last week's old entries # In theory, shouldn't do anything but being complete lastweek = datetime.now() - timedelta(days=7) graphs_collection.remove({'created': {"$lt": lastweek}}) graphs_data_collection.remove({'created': {"$lt": lastweek}}) graphs_links_collection.remove({'created': {"$lt": lastweek}}) graphs_docs_collection.remove({'created': {"$lt": lastweek}}) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete")
def main(): """ Begin Main... """ # The sources for which to remove expired entries # Infoblox is handled separately # Sonar RDNS is hard code below in a separate section # {"source_name": date_difference_in_months} sources = [{"name": "sonar_dns", "diff": -2}, {"name": "sonar_dns_saved", "diff": -2}, {"name": "ssl", "diff": -2}, {"name": "ssl_saved", "diff": -2}, {"name": "virustotal", "diff": -2}, {"name": "virustotal_saved", "diff": -2}, {"name": "UltraDNS", "diff": -2}, {"name": "UltraDNS_saved", "diff": -2}, {"name": "skms", "diff": -2}, {"name": "skms_saved", "diff": -2}, {"name": "marinus", "diff": -2}, {"name": "marinus_saved", "diff": -2}, {"name": "mx", "diff": -2}, {"name": "mx_saved", "diff": -2}, {"name": "common_crawl", "diff": -4}, {"name": "common_crawl_saved", "diff": -4}] now = datetime.now() print ("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() all_dns_collection = mongo_connector.get_all_dns_connection() dns_manager = DNSManager.DNSManager(mongo_connector) GDNS = GoogleDNS.GoogleDNS() zones = ZoneManager.get_distinct_zones(mongo_connector) jobs_collection = mongo_connector.get_jobs_connection() # Get the date for today minus two months d_minus_2m = monthdelta(datetime.now(), -2) print("Removing SRDNS as of: " + str(d_minus_2m)) # Remove the old records srdns_collection = mongo_connector.get_sonar_reverse_dns_connection() srdns_collection.remove({'updated': {"$lt": d_minus_2m}}) # Before completely removing old entries, make an attempt to see if they are still valid. # Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source. # Rather than throw away valid information, this will archive it. for entry in sources: removal_date = monthdelta(datetime.now(), entry['diff']) source = entry['name'] print("Removing " + source + " as of: " + str(removal_date)) last_domain = "" results = all_dns_collection.find({'sources': {"$size": 1}, 'sources.source': source, 'sources.updated': {"$lt": removal_date}}) for result in results: if result['fqdn'] != last_domain: last_domain = result['fqdn'] dns_result = GDNS.fetch_DNS_records(result['fqdn'], GDNS.DNS_TYPES[result['type']]) if dns_result != []: for dns_entry in dns_result: if is_tracked_zone(dns_entry['fqdn'], zones): new_entry={} new_entry['updated'] = datetime.now() new_entry['zone'] = result['zone'] new_entry['fqdn'] = dns_entry['fqdn'] new_entry['created'] = result['created'] new_entry['value'] = dns_entry['value'] new_entry['type'] = dns_entry['type'] new_entry['status'] = 'confirmed' if 'sonar_timestamp' in result: new_entry['sonar_timestamp'] = result['sonar_timestamp'] if source.endswith("_saved"): dns_manager.insert_record(new_entry, source) else: dns_manager.insert_record(new_entry, source + "_saved") dns_manager.remove_all_by_source_and_date(source, entry['diff']) # Record status jobs_collection.update_one({'job_name': 'remove_expired_entries'}, {'$currentDate': {"updated": True}, "$set": {'status': 'COMPLETE'}}) now = datetime.now() print("Complete: " + str(now))
import json import logging import os import requests import subprocess import sys import time from datetime import datetime from libs3 import DNSManager, MongoConnector, Rapid7, JobsManager, GoogleDNS from libs3.ZoneManager import ZoneManager from libs3.LoggingUtil import LoggingUtil mongo_connector = MongoConnector.MongoConnector() global_dns_manager = DNSManager.DNSManager(mongo_connector) def is_running(process): """ Is the provided process name is currently running? """ proc_list = subprocess.Popen(["pgrep", "-f", process], stdout=subprocess.PIPE) for proc in proc_list.stdout: if proc.decode('utf-8').rstrip() != str( os.getpid()) and proc.decode('utf-8').rstrip() != str( os.getppid()): return True return False
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) google_dns = GoogleDNS.GoogleDNS() jobs_manager = JobsManager.JobsManager(mongo_connector, 'extract_ssl_domains') jobs_manager.record_job_start() parser = argparse.ArgumentParser( description='Search TLS certificates for additional DNS names') parser.add_argument('--zgrab_version', default=2, type=int, choices=[1, 2], metavar="version", help='The version of ZGrab used to collect data') args = parser.parse_args() dns_names = [] round_two = [] zones = ZoneManager.get_distinct_zones(mongo_connector) # Collect the list of domains from the SSL Certificates extract_ct_certificate_names(dns_names, mongo_connector) # extract_censys_certificate_names(dns_names, mongo_connector) if args.zgrab_version == 1: extract_zgrab_certificate_names(logger, dns_names, mongo_connector) else: extract_zgrab2_certificate_names(logger, dns_names, mongo_connector) input_list = [] # Some SSL certificates are for multiple domains. # The tracked company may not own all domains. # Therefore, we filter to only the root domains that belong to the tracked company. logger.info("Pre-filter list: " + str(len(dns_names))) for hostname in dns_names: if not hostname.startswith("*"): zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) # Pause to prevent DoS-ing of Google's HTTPS DNS Service time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) if ip_addr['type'] == "cname" and is_tracked_zone( ip_addr['value'], zones): add_to_round_two(ip_addr['value'], round_two) else: logger.warning("Failed IP Lookup for: " + hostname) else: logger.warning("Failed match on zone for: " + hostname) else: logger.warning("Skipping wildcard: " + hostname) dead_dns_collection = mongo_connector.get_dead_dns_connection() # Some DNS records will be CNAME records pointing to other tracked domains. # This is a single level recursion to lookup those domains. logger.info("Round Two list: " + str(len(round_two))) for hostname in round_two: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) else: logger.warning("Failed IP Lookup for: " + hostname) original_record = dns_manager.find_one({"fqdn": hostname}, "ssl") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) else: logger.warning("Failed match on zone for: " + hostname) # Record all the results. dns_manager.remove_by_source("ssl") logger.info("List length: " + str(len(input_list))) for final_result in input_list: dns_manager.insert_record(final_result, "ssl") # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete")
class InfobloxDNSManager(object): # Make database connections MC = MongoConnector.MongoConnector() zone_collection = MC.get_zone_connection() APIH = APIHelper.APIHelper() IH = InfobloxHelper.InfobloxHelper() DNS_MGR = DNSManager.DNSManager(MC) next_page_id = None zone_queried = None record_type = None iblox_collection = None dns_value_mapper = { "mx": "mail_exchanger", "txt": "text", "a": "ipv4addr", "cname": "canonical", "aaaa": "ipv6addr", } _logger = None def _log(self): """ Get the log """ return logging.getLogger(__name__) def __get_record_type_url(self): """ Returns the url to be queried at infoblox to return the DNS information. Paging information is appended to the URL as per the condition satisfied. :return: string: URL to be queried """ paging_info = self.IH.get_pagination_params(self.next_page_id) url = self.IH.get_infoblox_base_url( self.zone_queried, self.record_type, ).format( return_fields="&_return_fields%2B=zone", paging_info=paging_info, ) return url def __get_previous_records(self): """ Retrieve the current data related to the zone. This is evaluated against the data that we receive in the latest script run to determine stale records. The data is stored as a list of _ref. """ self.previous_records = self.iblox_collection.distinct( "_ref", {"zone": self.zone_queried}) def __insert_dns_information(self, dns_information): """ Inserts the DNS information into the all_dns collection. For the 'host' records, iterate over all the ipv4addrs mentioned to get the data. :param dns_information: DNS data for the zone and 'record_type' """ del dns_information["_ref"] del dns_information["view"] del dns_information["infoblox_zone"] if self.record_type == "host": # In order to resolve multiple ipv4addrs for ipv4 in dns_information["ipv4addrs"]: dns_info = dict() dns_info["zone"] = dns_information["zone"] dns_info["type"] = "a" dns_info["value"] = ipv4["ipv4addr"] dns_info["fqdn"] = ipv4["host"] dns_info["status"] = "unknown" dns_info["created"] = datetime.now() self.DNS_MGR.insert_record(dns_info, "infoblox-host") else: # Removing the 'preference' key from the 'mx' records if self.record_type == "mx": del dns_information["preference"] dns_information["value"] = dns_information[self.dns_value_mapper[ self.record_type]] del dns_information[self.dns_value_mapper[self.record_type]] dns_information["fqdn"] = dns_information["name"] del dns_information["name"] dns_information["type"] = self.record_type dns_information["status"] = "unknown" dns_information["created"] = datetime.now() self.DNS_MGR.insert_record(dns_information, "infoblox-" + self.record_type) def __insert_records(self, insert_object): """ Inserts/Updates the dns information in the database. '_ref' uniquely identifies the resource. The data is injected into the individual collections belonging to the record_type and also into the all_dns collection. :param insert_object: Dictionary containing the details of the resource. """ dns_information = insert_object.copy() if not insert_object["_ref"] in self.previous_records: insert_object["created"] = datetime.now() insert_object["updated"] = datetime.now() self.iblox_collection.insert(insert_object) else: self.previous_records.remove(insert_object["_ref"]) insert_object["updated"] = datetime.now() self.iblox_collection.update_one({"_ref": insert_object["_ref"]}, {"$set": insert_object}) # Update DNS Information. self.__insert_dns_information(dns_information) def __infoblox_response_handler(self, response): """ Handles the API response. Incorrect JSON parsing is allowed upto 20 times post which the script exits. "next_page_id" holds the pagination information. :param response: Response object """ try: response_data = response.json() response_result = response_data["result"] except (ValueError, AttributeError) as err: if self.incorrect_response_json_allowed > 0: self._logger.warning( "Unable to parse response JSON for zone " + self.zone_queried) self.incorrect_response_json_allowed -= 1 else: self.APIH.handle_api_error( "Unable to parse response JSON for 20 zones: " + repr(err), "get_iblox_" + self.record_type.lower(), ) else: # Add the zone parameter to each record and insert for entry in response_result: entry["infoblox_zone"] = entry["zone"] entry["zone"] = self.zone_queried self.__insert_records(entry) if "next_page_id" in response_data: self.next_page_id = response_data["next_page_id"] @backoff.on_exception( backoff.expo, requests.exceptions.ConnectionError, max_tries=4, factor=10, on_backoff=APIH.connection_error_retry, ) def __backoff_api_retry(self): """ Makes API calls to Infoblox with exponential retry capabilities using 'backoff'. The API is retried 3 times in case of ConnectionError exception before the script exists. :return: """ return requests.get( (self.__get_record_type_url()), auth=HTTPBasicAuth(self.IH.IBLOX_UNAME, self.IH.IBLOX_PASSWD), verify="/etc/ssl/certs/ca-bundle.crt", timeout=120, ) def __infoblox_paginated_request(self): """ Makes paginated API calls to Infoblox. The API is retried 3 times in case of ConnectionError exception before the script exists. The script exists on encountering HTTPError or any other RequestException. On success, the next_page_id is set to None for the next API call. """ try: response = self.__backoff_api_retry() response.raise_for_status() except requests.exceptions.HTTPError as herr: self.APIH.handle_api_error(herr, "get_iblox_" + self.record_type.lower()) except requests.exceptions.RequestException as err: self.APIH.handle_api_error(err, "get_iblox_" + self.record_type.lower()) else: self.next_page_id = None self.__infoblox_response_handler(response) def get_infoblox_dns(self): """ Extracts the zones from the zone collection to query Infoblox. The API calls continue to be made for the zone till the next_page_id is set to None indicating no new results to be fetched. Post the retrieval of all the data, the archaic data for a zone is purged. """ zones = ZoneManager.get_zones_by_source(self.MC, "Infoblox") for zone in zones: self.zone_queried = zone self.next_page_id = None self.__get_previous_records() self.__infoblox_paginated_request() while self.next_page_id: self.__infoblox_paginated_request() self.IH.clean_collection(self.previous_records, self.iblox_collection) def __init__(self, record_type): self.record_type = record_type self.iblox_collection = self.MC.__getattribute__( self.IH.IBLOX_COLLECTIONS[self.record_type])() self.incorrect_response_json_allowed = self.APIH.INCORRECT_RESPONSE_JSON_ALLOWED self._logger = self._log()
def main(): """ Begin main... """ parser = argparse.ArgumentParser( description='Search the Common Crawl graph dataset for new domains') parser.add_argument('--url', metavar="URL", help='The URL for the latest vertices file') args = parser.parse_args() if args.url != None: CURRENT_FILE_LIST = args.url now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_collection = mongo_connector.get_jobs_connection() reversed_zones = ZoneManager.get_reversed_zones(mongo_connector) alphabet = list(string.digits + string.ascii_lowercase) # Create a dictionary of the zones grouped by their first letter # This will allow us to reduce the number of comparisons in the alphabetized CC files. grouped_zones = {} for letter in alphabet: grouped_zones[letter] = [] for zone in reversed_zones: first_letter = zone[0] grouped_zones[first_letter].append(zone) compressed_download_list = download_file(CURRENT_FILE_LIST) subprocess.check_call(["gunzip", "-f", compressed_download_list]) download_list = compressed_download_list.split(".")[:-1] list_file = ".".join(download_list) vertices_file_entries = open(list_file, "r") for entry in vertices_file_entries: # Download file vert_file_url = "http://commoncrawl.s3.amazonaws.com/" + entry.rstrip( "\n") compressed_vertices_file = download_file(vert_file_url) # Decompress file subprocess.check_call(["gunzip", "-f", compressed_vertices_file]) vertices_list = compressed_vertices_file.split(".")[:-1] vertices_file = ".".join(vertices_list) # Get the first and last line of the file (first_line, last_line) = get_first_and_last_line(vertices_file) # Get the first and last domain parts = first_line.split("\t") first_domain = parts[1].rstrip("\n") first_char = first_domain[0] parts = last_line.split("\t") last_domain = parts[1].rstrip("\n") last_char = last_domain[0] # Get the list of zones relevant to that range searchable_zones = get_zone_sublist(first_char, last_char, grouped_zones) # Parse file and insert matches parse_file(vertices_file, searchable_zones, dns_manager) subprocess.check_call(["rm", vertices_file]) # Remove all entries more than two months old # Note: This commented out because Common Crawl graph data is not additive. # dns_manager.remove_all_by_source_and_date("common_crawl", -4) jobs_collection.update_one({'job_name': 'common_crawl_graph'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) now = datetime.now() print("Ending: " + str(now))
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "extract_vt_domains") google_dns = GoogleDNS.GoogleDNS() jobs_manager.record_job_start() round_two = [] zones = ZoneManager.get_distinct_zones(mongo_connector) vt_collection = mongo_connector.get_virustotal_connection() vt_results = vt_collection.find({ "subdomains": { "$exists": True } }, { "zone": 1, "subdomains": 1 }).batch_size(30) input_list = [] # For each result found in the first pass across VirusTotal for result in vt_results: # Pause to prevent DoS-ing of Google's HTTPS DNS Service time.sleep(1) for hostname in result["subdomains"]: ips = google_dns.fetch_DNS_records(hostname) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr["fqdn"], zones) if temp_zone is not None: record = {"fqdn": ip_addr["fqdn"]} record["zone"] = temp_zone record["created"] = datetime.now() record["type"] = ip_addr["type"] record["value"] = ip_addr["value"] record["status"] = "unknown" input_list.append(record) if ip_addr["type"] == "cname" and is_tracked_zone( ip_addr["value"], zones): add_to_list(ip_addr["value"], round_two) else: logger.warning("Failed IP Lookup for: " + hostname) dead_dns_collection = mongo_connector.get_dead_dns_connection() # For each tracked CName result found in the first pass across VirusTotal logger.info("Round Two length: " + str(len(round_two))) for hostname in round_two: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr["fqdn"], zones) if temp_zone is not None: record = {"fqdn": ip_addr["fqdn"]} record["zone"] = temp_zone record["created"] = datetime.now() record["type"] = ip_addr["type"] record["value"] = ip_addr["value"] record["status"] = "unknown" input_list.append(record) else: original_record = dns_manager.find_one({"fqdn": hostname}, "virustotal") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) logger.warning("Failed IP Lookup for: " + hostname) else: logger.warning("Failed match on zone for: " + hostname) # Update the database dns_manager.remove_by_source("virustotal") logger.info("List length: " + str(len(input_list))) for final_result in input_list: dns_manager.insert_record(final_result, "virustotal") # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ # The sources for which to remove expired entries # Infoblox is handled separately # {"source_name": date_difference_in_months} sources = [{ "name": "sonar_dns", "diff": -2 }, { "name": "sonar_dns_saved", "diff": -2 }, { "name": "sonar_rdns", "diff": -2 }, { "name": "sonar_rdns_saved", "diff": -2 }, { "name": "ssl", "diff": -2 }, { "name": "ssl_saved", "diff": -2 }, { "name": "virustotal", "diff": -2 }, { "name": "virustotal_saved", "diff": -2 }, { "name": "UltraDNS", "diff": -2 }, { "name": "UltraDNS_saved", "diff": -2 }, { "name": "marinus", "diff": -2 }, { "name": "marinus_saved", "diff": -2 }, { "name": "mx", "diff": -2 }, { "name": "mx_saved", "diff": -2 }, { "name": "common_crawl", "diff": -4 }, { "name": "common_crawl_saved", "diff": -4 }] amass_diff = -2 now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() all_dns_collection = mongo_connector.get_all_dns_connection() dns_manager = DNSManager.DNSManager(mongo_connector) GDNS = GoogleDNS.GoogleDNS() ip_manager = IPManager.IPManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'remove_expired_entries') jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) # Get the date for today minus two months d_minus_2m = monthdelta(datetime.now(), -2) print("Removing SRDNS as of: " + str(d_minus_2m)) # Remove the old records srdns_collection = mongo_connector.get_sonar_reverse_dns_connection() srdns_collection.remove({'updated': {"$lt": d_minus_2m}}) ip_manager.delete_records_by_date(d_minus_2m) # Before completely removing old entries, make an attempt to see if they are still valid. # Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source. # Rather than throw away valid information, this will archive it. for entry in sources: removal_date = monthdelta(datetime.now(), entry['diff']) source = entry['name'] print("Removing " + source + " as of: " + str(removal_date)) last_domain = "" results = all_dns_collection.find({ 'sources': { "$size": 1 }, 'sources.source': source, 'sources.updated': { "$lt": removal_date } }) for result in results: if result['fqdn'] != last_domain: last_domain = result['fqdn'] lookup_int = get_lookup_int(result, GDNS) dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int) if dns_result != []: insert_current_results(dns_result, dns_manager, zones, result, source) dns_manager.remove_all_by_source_and_date(source, entry['diff']) # Process amass entries temp_sources = mongo_connector.perform_distinct(all_dns_collection, 'sources.source') amass_sources = [] for entry in temp_sources: if entry.startswith("amass:"): amass_sources.append(entry) for source in amass_sources: removal_date = monthdelta(datetime.now(), amass_diff) print("Removing " + source + " as of: " + str(removal_date)) last_domain = "" results = mongo_connector.perform_find( all_dns_collection, { 'sources': { "$size": 1 }, 'sources.source': source, 'sources.updated': { "$lt": removal_date } }) for result in results: if result['fqdn'] != last_domain: last_domain = result['fqdn'] lookup_int = get_lookup_int(result, GDNS) dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int) if dns_result != []: insert_current_results(dns_result, dns_manager, zones, result, source) dns_manager.remove_all_by_source_and_date(source, amass_diff) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now))
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, 'sonar_round_two') google_dns = GoogleDNS.GoogleDNS() jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) results = dns_manager.find_multiple({'type': 'cname'}, "sonar_dns") round_two = [] round_three = [] # Get all the CNAME values from all_dns and append them to round_two for result in results: if is_tracked_zone(result['value'], zones): round_two.append(result['value']) print("Round two pre-list: " + str(len(round_two))) dead_dns_collection = mongo_connector.get_dead_dns_connection() for value in round_two: is_present = dns_manager.find_count({'fqdn': value}, "sonar_dns") if is_present == 0: print(value + " not found") time.sleep(1) result = google_dns.fetch_DNS_records(value) if result == []: print("Unable to resolve") original_records = dns_manager.find_multiple({"value": value}, "sonar_dns") for record in original_records: check = dead_dns_collection.find({ 'fqdn': record['fqdn'] }).count() if check == 0: record.pop("_id") dead_dns_collection.insert(record) else: for entry in result: if is_tracked_zone(entry['fqdn'], zones): new_record = entry new_record['status'] = 'unconfirmed' new_record['zone'] = get_fld_from_value(value, '') new_record['created'] = datetime.now() if result[0]['type'] == "cname" and is_tracked_zone( entry['value'], zones): add_to_list(entry['value'], round_three) print("Found: " + value) if new_record['zone'] != '': dns_manager.insert_record(new_record, "marinus") # For each tracked CName result found in the first pass across Sonar DNS print("Round Three length: " + str(len(round_three))) for hostname in round_three: zone = get_fld_from_value(hostname, '') if zone != None and zone != '': ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: if is_tracked_zone(ip_addr['fqdn'], zones): record = {"fqdn": ip_addr['fqdn']} record['zone'] = get_fld_from_value( ip_addr['fqdn'], '') record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unconfirmed' dns_manager.insert_record(new_record, "marinus") else: original_record = dns_manager.find_one({"fqdn": hostname}, "marinus") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) print("Failed IP Lookup for: " + hostname) else: print("Failed match on zone for: " + hostname) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now))