def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print ("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dead_dns_collection = mongo_connector.get_dead_dns_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, 'dead_dns_cleanup') jobs_manager.record_job_start() google_dns = GoogleDNS.GoogleDNS() results = dead_dns_collection.find({}) for result in results: time.sleep(1) lookup_result = google_dns.fetch_DNS_records(result['fqdn']) if lookup_result == []: logger.info ("Removing " + result['fqdn']) dead_dns_collection.remove({"_id":ObjectId(result['_id'])}) # Record status jobs_manager.record_job_complete() now = datetime.now() print ("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) parser = argparse.ArgumentParser( description='Search Splunk logs for IP address') parser.add_argument('--collection_name', choices=["http_80", "http_443"], metavar="COLLECTION", required=True, help='The collection to upload to Splunk') args = parser.parse_args() mongo_connector = MongoConnector.MongoConnector() splunk_manager = SplunkHECManager.SplunkHECManager() jobs_manager = JobsManager.JobsManager(mongo_connector, "splunk_headers_upload") jobs_manager.record_job_start() if args.collection_name == "http_443": upload_zgrab_443(logger, splunk_manager, mongo_connector) elif args.collection_name == "http_80": upload_zgrab_80(logger, splunk_manager, mongo_connector) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() umbrella = Umbrella.Umbrella() zi = ZoneIngestor.ZoneIngestor() # Obtain the list of known email addresses and name servers from the config collection config_collection = mongo_connector.get_config_connection() res = config_collection.find({}) jobs_manager = JobsManager.JobsManager(mongo_connector, 'get_umbrella_whois') jobs_manager.record_job_start() # Perform a search for each email address for i in range(0, len(res[0]['DNS_Admins'])): search_umbrella_by_email(logger, res[0]['DNS_Admins'][i], umbrella, zi, jobs_manager) # Perform a search based on each name server for i in range(0, len(res[0]['Whois_Name_Servers'])): search_umbrella_by_nameserver(logger, res[0]['Whois_Name_Servers'][i], res[0]['Whois_Orgs'], umbrella, zi, jobs_manager) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Obtain the list of known email addresses from the config collection MC = MongoConnector.MongoConnector() PT = PassiveTotal.PassiveTotal() zi = ZoneIngestor.ZoneIngestor() config_collection = MC.get_config_connection() res = config_collection.find({}) jobs_manager = JobsManager.JobsManager(MC, 'get_passivetotal_data') jobs_manager.record_job_start() # Perform a search for each email address for i in range(0, len(res[0]['DNS_Admins'])): search_pt_email(logger, res[0]['DNS_Admins'][i], PT, zi, jobs_manager) for i in range(0, len(res[0]['Whois_Orgs'])): search_pt_org(logger, res[0]['Whois_Orgs'][i], PT, zi, jobs_manager) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print ("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() jobs_manager = JobsManager.JobsManager(mongo_connector, 'get_azure_data') jobs_manager.record_job_start() # Download the XML file req = requests.get(XML_LOCATION) if req.status_code != 200: logger.error("Bad Request") jobs_manager.record_job_error() exit(1) parser = MyHTMLParser() parser.feed(req.text) if parser.URL == "": logger.error("Unable to identify URL in Microsoft HTML") jobs_manager.record_job_error() exit(1) req = requests.get(parser.URL) if req.status_code != 200: logger.error("Bad Request") jobs_manager.record_job_error() exit(1) root = ET.fromstring(req.text) insert_json = {} insert_json['created'] = datetime.now() insert_json['prefixes'] = [] for region in root.findall('Region'): region_name = region.get("Name") for iprange in region.findall('IpRange'): cidr = iprange.get("Subnet") insert_json['prefixes'].append({'region': region_name, 'ip_prefix': cidr}) azure_ips = mongo_connector.get_azure_ips_connection() azure_ips.remove({}) azure_ips.insert(insert_json) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print ("Starting: " + str(now)) logger.info("Starting...") dns_types = {"a":1, "ns":2, "cname":5, "soa":6, "ptr":12, "hinfo": 13, "mx": 15, "txt":16, "aaaa":28, "srv":33, "naptr": 35, "ds": 43, "rrsig": 46, "dnskey": 48} mongo_connector = MongoConnector.MongoConnector() all_dns_collection = mongo_connector.get_all_dns_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, 'marinus_dns') jobs_manager.record_job_start() dns_manager = DNSManager.DNSManager(mongo_connector) zones = ZoneManager.get_distinct_zones(mongo_connector) google_dns = GoogleDNS.GoogleDNS() for zone in zones: time.sleep(1) for dtype, dnum in dns_types.items(): result = google_dns.fetch_DNS_records(zone, dnum) if result == []: logger.debug("No records found for " + zone) else: new_record = result[0] new_record['status'] = 'confirmed' new_record['zone'] = zone new_record['created'] = datetime.now() logger.debug ("Found " + dtype + " for: " + zone) dns_manager.insert_record(new_record, "marinus") logger.info("Starting SOA Search") soa_searches = find_sub_zones(all_dns_collection) for entry in soa_searches: time.sleep(1) result = google_dns.fetch_DNS_records(zone, dns_types['soa']) if result != []: new_record = result[0] new_record['status'] = 'confirmed' new_record['zone'] = get_fld_from_value(entry, '') new_record['created'] = datetime.now() logger.debug ("Found SOA: " + entry) if new_record['zone'] != '': dns_manager.insert_record(new_record, "marinus") jobs_manager.record_job_complete() now = datetime.now() print ("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = RemoteMongoConnector.RemoteMongoConnector() jobs_manager = JobsManager.JobsManager(mongo_connector, 'whois_lookups') jobs_manager.record_job_start() # Collect the tracked zones... zones = get_zones(mongo_connector) whois_collection = mongo_connector.get_whois_connection() for zone in zones: # Ensure the zone contains at least one dot. This is left over from an old bug. if zone.find(".") > 0: logger.debug(zone) zone_result = whois_collection.find_one({'zone': zone}) # If we haven't done a lookup in the past, try to collect the data. # A limit exists on the number of whois lookups you can perform so limit to new domains. if zone_result is None: do_whois_lookup(logger, zone, whois_collection) # The cap on the number of old entries to be updated. MAX_OLD_ENTRIES = 400 # Grab entries that haven't been updated in 3 months last_week = datetime.now() - timedelta(days=90, hours=1) zone_result = whois_collection.find({ "updated": { "$lte": last_week } }).batch_size(10) i = 0 for result in zone_result: do_whois_lookup(logger, result["zone"], whois_collection) i = i + 1 # Chances are that a lot of the entries were inserted on the same day. # This helps break updating old entries across different runs. if i > MAX_OLD_ENTRIES: break # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def __init__(self, thread_id, q, port, command, zone, zgrab_collection): threading.Thread.__init__(self) self.thread_id = thread_id self.port = port self.zgrab_collection = zgrab_collection self.zone = zone self.run_command = command self.q = q self.logger = LoggingUtil.create_log(__name__)
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print ("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zone_ingestor = ZoneIngestor.ZoneIngestor() jobs_manager = JobsManager.JobsManager(mongo_connector, 'get_route53') jobs_manager.record_job_start() current_zones = ZoneManager.get_distinct_zones(mongo_connector) # For cases with multiple R53 accounts, include the account id for reference sts = boto3.client('sts') account_id = sts.get_caller_identity()["Arn"].split(':')[4] r53_source = "R53:" + str(account_id) r53_client = boto3.client('route53') r53_domains = r53_client.list_hosted_zones() r53_zone_list = [] while r53_domains != {}: for zone_data in r53_domains['HostedZones']: # Only add public zones if zone_data['Config']['PrivateZone'] == False: r53_zone_list.append(zone_data) if r53_domains['IsTruncated'] == True: r53_domains = r53_client.list_domains(Marker=r53_domains['NextMarker']) else: r53_domains = {} for zone_data in r53_zone_list: # Double check that this is not a new zone zone_name = zone_data['Name'][:-1] if zone_name not in current_zones: logger.info("Creating zone: " + zone_name) zone_ingestor.add_zone(zone_data['Name'], r53_source) # Add hosts to the zone update_records(r53_client, dns_manager, zone_data, r53_source) # Record status jobs_manager.record_job_complete() now = datetime.now() print ("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "get_external_cnames") jobs_manager.record_job_start() groups = {} # Collect zones zone_results = ZoneManager.get_distinct_zones(mongo_connector) zones = [] for zone in zone_results: if zone.find(".") >= 0: zones.append(zone) # Collect the all_dns cnames. logger.info("Starting All DNS...") all_dns_recs = dns_manager.find_multiple({"type": "cname"}, None) for srec in all_dns_recs: if not is_tracked_zone(srec["value"], zones): add_to_list( get_fld_from_value(srec["value"], srec["zone"]), srec["fqdn"], srec["value"], srec["zone"], groups, ) # Update the database tpds_collection = mongo_connector.get_tpds_connection() tpds_collection.delete_many({}) for key in groups.keys(): tpds_collection.insert_one(groups[key]) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ This function extract the IP address ranges from the TXT records and stores them in gcp_ips collection within the database. """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() gcp_collection = mongo_connector.get_gcp_ips_connection() google_dns = GoogleDNS.GoogleDNS() jobs_manager = JobsManager.JobsManager(mongo_connector, 'get_gcp_ranges') jobs_manager.record_job_start() ip_ranges = recursive_search(logger, "_cloud-netblocks.googleusercontent.com", google_dns) ipv4_ranges = [] ipv6_ranges = [] for entry in ip_ranges: parts = entry.split(":", 1) if parts[0] == "ip4" and parts[1] not in ipv4_ranges: ipv4_ranges.append({"ip_prefix": parts[1]}) elif parts[0] == "ip6" and parts[1] not in ipv6_ranges: ipv6_ranges.append({"ipv6_prefix": parts[1]}) else: logger.warning("Unrecognized data: " + entry) new_data = {} new_data['prefixes'] = ipv4_ranges new_data['ipv6_prefixes'] = ipv6_ranges new_data['created'] = now gcp_collection.remove({}) gcp_collection.insert(new_data) jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
class MyHTMLParser(HTMLParser): """ Create a subclass and override the handler methods. """ URL = "" logger = LoggingUtil.create_log(__name__) def handle_starttag(self, tag, attrs): found = False if tag == "a": for attr in attrs: if attr[0] == "class" and attr[1] == "mscom-link failoverLink": found = True if found: for attr in attrs: if attr[0] == "href": self.logger.info(attr[1]) self.URL = attr[1]
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() splunk_query_manager = SplunkQueryManager.SplunkQueryManager() splunk_collection = mongo_connector.get_splunk_connection() dns_manager = DNSManager.DNSManager(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "get_splunk_data") jobs_manager.record_job_start() logger.info ("Starting Splunk Query") results_per_page = 100 # Put your custom Splunk search query here. results = splunk_query_manager.do_search('search index=...', results_per_page) logger.info ("Processing " + str(splunk_query_manager.RESULTCOUNT) + " results") parse_splunk_results(logger, results, dns_manager, splunk_collection) while True: results = splunk_query_manager.get_next_page() if results is None: break parse_splunk_results(logger, results, dns_manager, splunk_collection) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) print("Starting: " + str(datetime.now())) logger.info("Starting...") # Make database connections mc = MongoConnector.MongoConnector() jobs_manager = JobsManager.JobsManager(mc, "get_iblox_txt") jobs_manager.record_job_start() idm = InfobloxDNSManager.InfobloxDNSManager("txt") idm.get_infoblox_dns() # Record status jobs_manager.record_job_complete() print("Ending: " + str(datetime.now())) logger.info("Complete.")
def get_ultradns_zones(self): """ Extracts the zones listing from UltraDNS in a paginated manner. """ self._logger = LoggingUtil.create_log(__name__) print("Starting: " + str(datetime.now())) self._logger.info("Starting...") self.UH.jobs_manager.record_job_start() # Part of clean_collection code. # self.UH.get_previous_zones() self.__paginated_ultradns_zones_request() while self.UH.offset: self.__paginated_ultradns_zones_request() # Record status self.UH.jobs_manager.record_job_complete() print("Ending: " + str(datetime.now())) self._logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) print("Starting: " + str(datetime.now())) logger.info("Starting...") # Make database connections mc = MongoConnector.MongoConnector() jobs_manager = JobsManager.JobsManager(mc, 'get_infoblox_cname_extattrs') jobs_manager.record_job_start() iem = InfobloxExtattrManager.InfobloxExtattrManager('cname') iem.get_infoblox_extattr() # Record status jobs_manager.record_job_complete() print("Ending: " + str(datetime.now())) logger.info("Complete.")
def main(): """ Begin main... """ logger = LoggingUtil.create_log(__name__) # Make database connections mongo_connector = MongoConnector.MongoConnector() now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") jobs_manager = JobsManager.JobsManager(mongo_connector, 'get_aws_data') jobs_manager.record_job_start() # Download the JSON file req = requests.get(JSON_LOCATION) if req.status_code != 200: logger.error("Bad Request") jobs_manager.record_job_error() exit(1) # Convert the response to JSON json_data = json.loads(req.text) # Replace the old entries with the new entries aws_collection = mongo_connector.get_aws_ips_connection() aws_collection.remove({}) aws_collection.insert(json_data) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") jobs_manager = JobsManager.JobsManager(mongo_connector, 'remote_download') jobs_manager.record_job_start() remote_jobs_collection = rm_connector.get_jobs_connection() # Check the status of the Censys job on the remote database status = remote_jobs_collection.find_one({'job_name': 'censys'}) if status is not None and 'status' in status and status[ 'status'] != jobs_manager.COMPLETE: logger.info("Censys scans status is not COMPLETE") elif status is not None and 'status' in status and status[ 'status'] == jobs_manager.COMPLETE: # Get connections to the relevant collections. censys_collection = mongo_connector.get_zgrab_443_data_connection() remote_censys_collection = rm_connector.get_zgrab_443_data_connection() download_censys_scan_info(censys_collection, remote_censys_collection) # Tell the remote database that is safe to start processing the next Censys file remote_jobs_collection.update_one({'job_name': 'censys'}, { '$currentDate': { "updated": True }, "$set": { 'status': jobs_manager.READY } }) # Get connections to the relevant HTTPS collections. zgrab_443_data_collection = mongo_connector.get_zgrab_443_data_connection() remote_zgrab_443_data_collection = rm_connector.get_zgrab_443_data_connection( ) download_zgrab_info(logger, zgrab_443_data_collection, remote_zgrab_443_data_collection) # Get connections to the relevant HTTP collections. zgrab_80_data_collection = mongo_connector.get_zgrab_80_data_connection() remote_zgrab_80_data_collection = rm_connector.get_zgrab_80_data_connection( ) download_zgrab_info(logger, zgrab_80_data_collection, remote_zgrab_80_data_collection) # Get connections to the relevant port collections. zgrab_port_data_collection = mongo_connector.get_zgrab_port_data_connection( ) remote_zgrab_port_data_collection = rm_connector.get_zgrab_port_data_connection( ) download_zgrab_port_info(logger, zgrab_port_data_collection, remote_zgrab_port_data_collection) # Download latest whois information status = remote_jobs_collection.find_one({'job_name': 'whois_lookups'}) if status['status'] == jobs_manager.COMPLETE: whois_collection = mongo_connector.get_whois_connection() remote_whois_collection = rm_connector.get_whois_connection() download_whois_data(logger, whois_collection, remote_whois_collection) remote_jobs_collection.update({'job_name': 'whois'}, {'$set': { 'status': jobs_manager.READY }}) # Download the status of the remote jobs download_jobs_status(logger, jobs_manager._jobs_collection, remote_jobs_collection) # Update the local jobs database to done jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) google_dns = GoogleDNS.GoogleDNS() jobs_manager = JobsManager.JobsManager(mongo_connector, 'extract_ssl_domains') jobs_manager.record_job_start() parser = argparse.ArgumentParser( description='Search TLS certificates for additional DNS names') parser.add_argument('--zgrab_version', default=2, type=int, choices=[1, 2], metavar="version", help='The version of ZGrab used to collect data') args = parser.parse_args() dns_names = [] round_two = [] zones = ZoneManager.get_distinct_zones(mongo_connector) # Collect the list of domains from the SSL Certificates extract_ct_certificate_names(dns_names, mongo_connector) # extract_censys_certificate_names(dns_names, mongo_connector) if args.zgrab_version == 1: extract_zgrab_certificate_names(logger, dns_names, mongo_connector) else: extract_zgrab2_certificate_names(logger, dns_names, mongo_connector) input_list = [] # Some SSL certificates are for multiple domains. # The tracked company may not own all domains. # Therefore, we filter to only the root domains that belong to the tracked company. logger.info("Pre-filter list: " + str(len(dns_names))) for hostname in dns_names: if not hostname.startswith("*"): zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) # Pause to prevent DoS-ing of Google's HTTPS DNS Service time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) if ip_addr['type'] == "cname" and is_tracked_zone( ip_addr['value'], zones): add_to_round_two(ip_addr['value'], round_two) else: logger.warning("Failed IP Lookup for: " + hostname) else: logger.warning("Failed match on zone for: " + hostname) else: logger.warning("Skipping wildcard: " + hostname) dead_dns_collection = mongo_connector.get_dead_dns_connection() # Some DNS records will be CNAME records pointing to other tracked domains. # This is a single level recursion to lookup those domains. logger.info("Round Two list: " + str(len(round_two))) for hostname in round_two: zone = get_tracked_zone(hostname, zones) if zone != None: ips = google_dns.fetch_DNS_records(hostname) time.sleep(1) if ips != []: for ip_addr in ips: temp_zone = get_tracked_zone(ip_addr['fqdn'], zones) if temp_zone is not None: record = {"fqdn": ip_addr['fqdn']} record['zone'] = temp_zone record['created'] = datetime.now() record['type'] = ip_addr['type'] record['value'] = ip_addr['value'] record['status'] = 'unknown' input_list.append(record) else: logger.warning("Failed IP Lookup for: " + hostname) original_record = dns_manager.find_one({"fqdn": hostname}, "ssl") if original_record != None: original_record.pop("_id") dead_dns_collection.insert(original_record) else: logger.warning("Failed match on zone for: " + hostname) # Record all the results. dns_manager.remove_by_source("ssl") logger.info("List length: " + str(len(input_list))) for final_result in input_list: dns_manager.insert_record(final_result, "ssl") # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Obtain the list of known email addresses from the config collection mongo_connector = MongoConnector.MongoConnector() whois_collection = mongo_connector.get_whois_connection() all_dns_collection = mongo_connector.get_all_dns_connection() zones_collection = mongo_connector.get_zone_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, 'mark_expired') jobs_manager.record_job_start() # Grab all zones that are not expired of false_positives # Also exclude any that were recently created since they won't have data yet date_delta = datetime.today() - timedelta(days=30) zones = zones_collection.distinct( 'zone', { 'created': { "$lt": date_delta }, 'status': { "$nin": [ZoneManager.EXPIRED, ZoneManager.FALSE_POSITIVE] } }) # The Python Whois library is hit and miss with some international zones. # For now, this script focuses on the most popular TLDs. new_zones = get_primary_zones(logger, zones) expired_list = [] for zone in new_zones: if whois_collection.find({'zone': zone}).count() == 0: # Assume it is expired if there is no longer a whois record present expired_list.append(zone) for zone in expired_list: if all_dns_collection.find({'zone': zone}).count() > 0: # This may be a case where the Python Whois library failed # and the zone is still active. logger.debug("DNS records still exist for " + zone) expired_list.remove(zone) zone_manager = ZoneManager(mongo_connector) # Need to get this list before setting zones to expired in order to avoid a recursion problem. already_expired = zone_manager.get_zones_by_status(ZoneManager.EXPIRED) possibly_renewed = [] for zone in already_expired: if whois_collection.find({'zone': zone}).count() == 1: possibly_renewed.append(zone) for zone in expired_list: logger.debug("Expiring: " + zone) zone_manager.set_status(zone, ZoneManager.EXPIRED, "mark_expired.py") # Get the list of known registering entities. # This will only work for some whois lookups since Python Whois doesn't get # a valid org for all lookups and some have privacy enabled. config_collection = mongo_connector.get_config_connection() result = config_collection.find({}, { 'Whois_Orgs': 1, 'Whois_Name_Servers': 1 }) orgs = result[0]['Whois_Orgs'] name_servers = [] if 'Whois_Name_Servers' in result[0]: name_servers = result[0]['Whois_Name_Servers'] logger.debug(str(name_servers)) for zone in possibly_renewed: # We need to be careful of automatically marking something renewed # since it could have been registered by someone else. if whois_collection.find({ 'zone': zone, 'org': { "$in": orgs } }).count() == 1: logger.warning("ATTENTION: " + zone + " has been renewed based on org") zone_manager.set_status(zone, ZoneManager.UNCONFIRMED, "mark_expired.py") else: result = whois_collection.find({'zone': zone}, { 'name_servers': 1, "_id": 0 }) found = 0 if result is not None and 'name_servers' in result[0] and result[ 0]['name_servers'] is not None: for entry in result[0]['name_servers']: if entry.lower() in name_servers: logger.warning( "ATTENTION: " + zone + " has been renewed based on name servers") zone_manager.set_status(zone, ZoneManager.UNCONFIRMED, "mark_expired.py") found = 1 break if found == 0: result = whois_collection.find({'zone': zone}, { 'name_server_groups': 1, "_id": 0 }) if result is not None and 'name_server_groups' in result[ 0] and result[0]['name_server_groups'] is not None: for entry in result[0]['name_server_groups']: if entry.lower() in name_servers: logger.warning( "ATTENTION: " + zone + " has been renewed based on name server_groups" ) zone_manager.set_status(zone, ZoneManager.UNCONFIRMED, "mark_expired.py") found = 1 break if found == 0: logger.warning(zone + " has been renewed by an unknown entity") # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Make database connections mongo_connector = MongoConnector.MongoConnector() ct_collection = mongo_connector.get_certificate_transparency_connection() config_collection = mongo_connector.get_config_connection() x509parser = X509Parser.X509Parser() zones = ZoneManager.get_distinct_zones(mongo_connector) result = config_collection.find_one({}, {'SSL_Orgs': 1, "_id": 0}) ssl_orgs = result['SSL_Orgs'] # Defaults save_location = '/mnt/workspace/' download_method = 'dbAndSave' save_type = "PEM" parser = argparse.ArgumentParser( description='Download certificate information from the provide CT Log.' ) parser.add_argument( '--log_source', required=True, help= 'Indicates which log to query based on values in the x509Parser library' ) parser.add_argument( '--include_precerts', action="store_true", help='Include pre-certificates which are not finalized') parser.add_argument( '--download_methods', choices=['dbAndSave', 'dbOnly'], default=download_method, help= 'Indicates whether to download the raw files or just save to the database' ) parser.add_argument( '--starting_index', required=False, default=-1, type=int, help='Force the script to start at specific index within the log.') parser.add_argument( '--cert_save_location', required=False, default=save_location, help= 'Indicates where to save the certificates on disk when choosing dbAndSave' ) parser.add_argument( '--save_type', choices=['PEM', 'ASN1'], default=save_type, help='Indicates which format to use for the data. The default is PEM') args = parser.parse_args() source = args.log_source try: ct_log_map = x509parser.CT_LOG_MAP[source] except: logger.error("ERROR: UNKNOWN LOG SOURCE: " + source) exit(1) if args.cert_save_location: save_location = args.cert_save_location if not save_location.endswith("/"): save_location = save_location + "/" if args.download_methods: download_method = args.download_methods check_save_location(save_location, source) if args.save_type: save_type = args.save_type jobs_manager = JobsManager.JobsManager(mongo_connector, "ct_log-" + source) jobs_manager.record_job_start() if args.starting_index == -1: starting_index = fetch_starting_index(ct_collection, source) else: starting_index = args.starting_index logger.info("Starting Index: " + str(starting_index)) sth_data = fetch_sth(logger, "https://" + ct_log_map['url']) logger.info("Tree size: " + str(sth_data['tree_size'])) current_index = starting_index while current_index < sth_data['tree_size']: ending_index = current_index + 256 if ending_index > sth_data['tree_size']: ending_index = sth_data['tree_size'] logger.debug("Checking from index: " + str(current_index) + " to index " + str(ending_index)) certs = fetch_certificate_batch(logger, "https://" + ct_log_map['url'], current_index, ending_index) for entry in certs['entries']: der_cert, cert_type = get_cert_from_leaf(logger, entry['leaf_input']) if der_cert is None and cert_type == 1 and not args.include_precerts: current_index = current_index + 1 continue elif der_cert is None and cert_type == 0: current_index = current_index + 1 continue elif der_cert is None and cert_type == 1: der_cert = get_cert_from_extra_data(entry['extra_data']) cert = x509parser.parse_data(der_cert, source) if cert is None: logger.warning("Skipping certificate index: " + str(current_index)) current_index = current_index + 1 continue if cert_type == 1: cert['ct_log_type'] = "PRE-CERTIFICATE" else: cert['ct_log_type'] = "CERTIFICATE" cert_zones = check_zone_relevancy(cert, zones) if check_org_relevancy(cert, ssl_orgs) or cert_zones != []: cert[source + "_id"] = current_index cert['zones'] = cert_zones logger.info("Adding " + source + " id: " + str(current_index) + " SHA256: " + cert['fingerprint_sha256']) insert_certificate(cert, source, ct_collection, cert_zones) if download_method == 'dbAndSave': write_file(logger, cert, save_location, save_type, source) current_index = current_index + 1 # Set isExpired for any entries that have recently expired. ct_collection.update( { "not_after": { "$lt": datetime.utcnow() }, "isExpired": False }, {"$set": { "isExpired": True }}, multi=True) jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") parser = argparse.ArgumentParser( description= 'Send specific collections to the remote MongoDB. If no arguments are provided, then all data is mirrored.' ) parser.add_argument('--send_zones', action="store_true", required=False, help='Send IP zones') parser.add_argument('--send_ip_zones', action="store_true", required=False, help='Send IP zones') parser.add_argument('--send_third_party_zones', action="store_true", required=False, help='Send AWS, Azure, etc.') parser.add_argument('--send_config', action="store_true", required=False, help='Send configs') parser.add_argument('--send_dns_records', action="store_true", required=False, help='Send DNS records') args = parser.parse_args() send_all = False if len(sys.argv) == 1: send_all = True mongo_connector = MongoConnector.MongoConnector() remote_mongo_connector = RemoteMongoConnector.RemoteMongoConnector() jobs_manager = JobsManager.JobsManager(mongo_connector, 'send_remote_server') jobs_manager.record_job_start() if send_all or args.send_zones: zone_list = update_zones(logger, mongo_connector, remote_mongo_connector, True) else: zone_list = update_zones(logger, mongo_connector, remote_mongo_connector, False) if send_all or args.send_ip_zones: update_ip_zones(logger, mongo_connector, remote_mongo_connector) if send_all or args.send_third_party_zones: update_aws_cidrs(logger, mongo_connector, remote_mongo_connector) update_azure_cidrs(logger, mongo_connector, remote_mongo_connector) update_akamai_cidrs(logger, mongo_connector, remote_mongo_connector) update_gcp_cidrs(logger, mongo_connector, remote_mongo_connector) if send_all or args.send_config: update_config(logger, mongo_connector, remote_mongo_connector) if send_all or args.send_dns_records: update_all_dns(logger, mongo_connector, remote_mongo_connector, zone_list) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") parser = argparse.ArgumentParser( description="Send specific collections to the remote MongoDB. If no arguments are provided, then all data is mirrored." ) parser.add_argument( "--send_zones", action="store_true", required=False, help="Send IP zones" ) parser.add_argument( "--send_ip_zones", action="store_true", required=False, help="Send IP zones" ) parser.add_argument( "--send_third_party_zones", action="store_true", required=False, help="Send AWS, Azure, etc.", ) parser.add_argument( "--send_config", action="store_true", required=False, help="Send configs" ) parser.add_argument( "--send_dns_records", action="store_true", required=False, help="Replace all DNS records", ) parser.add_argument( "--send_dns_diff", action="store_true", required=False, help="Send new DNS records", ) parser.add_argument( "--date_diff", default=2, type=int, help="The number of days used for identifying new records in send_dns_diff", ) args = parser.parse_args() send_all = False if len(sys.argv) == 1: send_all = True mongo_connector = MongoConnector.MongoConnector() remote_mongo_connector = RemoteMongoConnector.RemoteMongoConnector() jobs_manager = JobsManager.JobsManager(mongo_connector, "send_remote_server") jobs_manager.record_job_start() if send_all or args.send_zones: try: zone_list = update_zones( logger, mongo_connector, remote_mongo_connector, True ) except: logger.error( "Could not communicate with the remote database when sending zones" ) jobs_manager.record_job_error() exit(1) else: zone_list = update_zones(logger, mongo_connector, remote_mongo_connector, False) if send_all or args.send_ip_zones: try: update_ip_zones(logger, mongo_connector, remote_mongo_connector) except: logger.error( "Could not communicate with the remote database when sending IP zones" ) jobs_manager.record_job_error() exit(1) if send_all or args.send_third_party_zones: try: update_aws_cidrs(logger, mongo_connector, remote_mongo_connector) update_azure_cidrs(logger, mongo_connector, remote_mongo_connector) update_akamai_cidrs(logger, mongo_connector, remote_mongo_connector) update_gcp_cidrs(logger, mongo_connector, remote_mongo_connector) except: logger.error( "Could not communicate with the remote database when sending third-party zones" ) jobs_manager.record_job_error() exit(1) if send_all or args.send_config: try: update_config(logger, mongo_connector, remote_mongo_connector) except: logger.error( "Could not communicate with the remote database when sending config data" ) jobs_manager.record_job_error() exit(1) # This will completely repopulate the DNS records table. if args.send_dns_records is not False: try: update_all_dns(logger, mongo_connector, remote_mongo_connector, zone_list) except: logger.error( "Could not communicate with the remote database when sending DNS records" ) jobs_manager.record_job_error() exit(1) # If you have a large data set, then you may only want to send updated records if send_all or args.send_dns_diff: try: update_all_dns_diff_mode( logger, mongo_connector, remote_mongo_connector, zone_list, args.date_diff, ) except: logger.error( "Could not communicate with the remote database when sending DNS diff records" ) jobs_manager.record_job_error() exit(1) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main() """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = MongoConnector.MongoConnector() mongo_ct = mongo_connector.get_certificate_transparency_connection() cert_graphs_collection = mongo_connector.get_cert_graphs_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, "create_cert_graphs") jobs_manager.record_job_start() zones = ZoneManager.get_distinct_zones(mongo_connector) parser = argparse.ArgumentParser( description= "Creates and stores certificate graphs in the database based on one or more sources." ) parser.add_argument( "--check_censys", action="store_true", default=False, required=False, help="Whether to check the Censys collection in the database", ) parser.add_argument( "--check_443_scans", action="store_true", default=False, required=False, help="Whether to check the zgrab collection in the database", ) parser.add_argument( "--check_ct_scans", action="store_true", default=False, required=False, help="Whether to check the CT collection in the database", ) parser.add_argument( "--zgrab_version", default=2, type=int, choices=[1, 2], metavar="version", help="The version of ZGrab used to collect data", ) args = parser.parse_args() if args.check_censys is True: censys_collection = mongo_connector.get_censys_connection() if args.check_443_scans is True: zgrab_collection = mongo_connector.get_zgrab_443_data_connection() for zone in zones: logger.info("Creating: " + zone) graph = nx.DiGraph() certs_list = {} if args.check_ct_scans: certs_list = get_current_ct_certificates(mongo_ct, zone) if args.check_censys: certs_list = add_censys_certificates(censys_collection, zone, certs_list) if args.check_443_scans: if args.zgrab_version == 1: certs_list = add_terminal_zgrab_certificates( zgrab_collection, zone, certs_list) certs_list = add_initial_zgrab_certificates( zgrab_collection, zone, certs_list) else: certs_list = add_terminal_zgrab2_certificates( zgrab_collection, zone, certs_list) certs_list = add_initial_zgrab2_certificates( zgrab_collection, zone, certs_list) graph = create_nodes(graph, mongo_connector, zone, certs_list) data = json_graph.node_link_data(graph) my_data = {} my_data["links"] = data["links"] my_data["nodes"] = data["nodes"] my_data["zone"] = zone my_data["created"] = datetime.now() cert_graphs_collection.delete_one({"zone": zone}) mongo_connector.perform_insert(cert_graphs_collection, my_data) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin main... """ logger = LoggingUtil.create_log(__name__) if is_running("get_censys_files.py"): """ Check to see if a download is in process... """ logger.warning("Can't run due to get_files running. Goodbye!") exit(0) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ logger.warning("I am already running! Goodbye!") exit(0) # Make the relevant database connections RMC = RemoteMongoConnector.RemoteMongoConnector() ip_manager = IPManager.IPManager(RMC) # Verify that the get_files script has a recent file in need of parsing. jobs_collection = RMC.get_jobs_connection() status = jobs_collection.find_one({"job_name": "censys"}) if status["status"] != "DOWNLOADED": logger.warning("The status is not set to DOWNLOADED. Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Collect the list of available zones zones = ZoneManager.get_distinct_zones(RMC) logger.info("Zones: " + str(len(zones))) # Get the current configuration information for Marinus. config_collection = RMC.get_config_connection() configs = config_collection.find({}) orgs = [] for org in configs[0]["SSL_Orgs"]: orgs.append(org) logger.info("Orgs: " + str(len(orgs))) # Obtain the name of the decompressed file. filename_f = open(FILENAME_FILE, "r") decompressed_file = filename_f.readline() filename_f.close() # For manual testing: decompressed_file = "ipv4.json" logger.info("Beginning file processing...") # Remove old results from the database results_collection = RMC.get_results_connection() results_collection.delete_many({}) all_dns_collection = RMC.get_all_dns_connection() try: with open(decompressed_file, "r") as dec_f: for line in dec_f: try: entry = json.loads(line) """ Does the SSL certificate match a known organization? Is the IP address in a known CIDR? Is the IP address recorded in Splunk? """ if (check_in_org(entry, orgs) or ip_manager.is_tracked_ip(entry["ip"]) or ip_manager.find_splunk_data(entry["ip"], "AWS") is not None or ip_manager.find_splunk_data( entry["ip"], "AZURE") is not None): entry["zones"] = check_in_zone(entry, zones) entry["aws"] = ip_manager.is_aws_ip(entry["ip"]) entry["azure"] = ip_manager.is_azure_ip(entry["ip"]) (domains, zones) = lookup_domain(entry, zones, all_dns_collection) if len(domains) > 0: entry["domains"] = domains if len(zones) > 0: for zone in zones: if zone not in entry["zones"]: entry["zones"].append(zone) insert_result(entry, results_collection) # else: # #This will add days to the amount of time necessary to scan the file. # matched_zones = check_in_zone(entry, zones) # if matched_zones != []: # entry['zones'] = matched_zones # entry['aws'] = ip_manager.is_aws_ip(entry['ip']) # entry['azure'] = ip_manager.is_azure_ip(entry['ip']) # insert_result(entry, results_collection) except ValueError as err: logger.error("Value Error!") logger.error(str(err)) except: logger.error("Line unexpected error: " + str(sys.exc_info()[0])) logger.error("Line unexpected error: " + str(sys.exc_info()[1])) except IOError as err: logger.error("I/O error({0}): {1}".format(err.errno, err.strerror)) exit(1) except: logger.error("Unexpected error: " + str(sys.exc_info()[0])) logger.error("Unexpected error: " + str(sys.exc_info()[1])) exit(1) # Indicate that the processing of the job is complete and ready for download to Marinus jobs_collection.update_one( {"job_name": "censys"}, { "$currentDate": { "updated": True }, "$set": { "status": "COMPLETE" } }, ) now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def __init__(self): self._logger = LoggingUtil.create_log(__name__) self.incorrect_response_json_allowed = self.APIH.INCORRECT_RESPONSE_JSON_ALLOWED self.get_infoblox_zones()
self._logger.info("Starting....") self.job_manager = JobsManager.JobsManager(self.MC, 'get_iblox_alpha_zones') self.job_manager.record_job_start() self.__get_previous_zones() for alphabet in self.alphabets: self.alphabet_queried = alphabet self.next_page_id = None self.__infoblox_paginated_request() while self.next_page_id: self.__infoblox_paginated_request() self.__clean_collection() # Record status self.job_manager.record_job_complete() print("Ending: " + str(datetime.now())) self._logger.info("Complete") def __init__(self): self._logger = LoggingUtil.create_log(__name__) self.incorrect_response_json_allowed = self.APIH.INCORRECT_RESPONSE_JSON_ALLOWED self.get_infoblox_zones() if __name__ == '__main__': logger = LoggingUtil.create_log(__name__) IZ = InfobloxZone()
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Set up the common objects mongo_connector = MongoConnector.MongoConnector() ct_collection = mongo_connector.get_certificate_transparency_connection() zones = ZoneManager.get_distinct_zones(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "get_crt_sh") jobs_manager.record_job_start() save_location = "/mnt/workspace/crt_sh" download_method = 'dbAndSave' parser = argparse.ArgumentParser( description='Download DNS and/or certificate information from crt.sh.') parser.add_argument( '--fetch_dns_records', action='store_true', help='Indicates whether to add DNS entries to the database') parser.add_argument( '--download_methods', choices=['dbAndSave', 'dbOnly'], default=download_method, help= 'Indicates whether to download the raw files or just record in the database.' ) parser.add_argument( '--cert_save_location', required=False, default=save_location, help= 'Indicates where to save the certificates on disk when choosing dbAndSave' ) args = parser.parse_args() if args.cert_save_location: save_location = args.cert_save_location if not save_location.endswith("/"): save_location = save_location + "/" if args.download_methods == 'dbAndSave': check_save_location(save_location) for zone in zones: # Pace out requests so as not to DoS crt.sh and Google DNS time.sleep(5) # This could be done with backoff but we don't want to be overly aggressive. json_result = make_https_request( logger, "https://crt.sh/?q=%25." + zone + "&output=json") if json_result is None: logger.warning("Can't find result for: " + zone) json_result = "{}" json_data = json.loads(json_result) new_names = [] new_ids = [] for entry in json_data: if entry['id'] not in new_ids: new_ids.append(entry['id']) if "*" not in entry["name_value"] and entry[ "name_value"] not in new_names: new_names.append(entry["name_value"]) if args.fetch_dns_records: add_new_domain_names(new_names, zones, mongo_connector) if args.download_methods == "dbAndSave": add_new_certificate_values(logger, new_ids, ct_collection, zones, save_location) elif args.download_methods == "dbOnly": add_new_certificate_values(logger, new_ids, ct_collection, zones, None) # Set isExpired for any entries that have recently expired. ct_collection.update( { "not_after": { "$lt": datetime.utcnow() }, "isExpired": False }, {"$set": { "isExpired": True }}, multi=True) jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Beging Main... """ global global_exit_flag global global_retest_list global global_sleep_time global global_queue_size global global_zgrab_path logger = LoggingUtil.create_log(__name__) global_retest_list = [] parser = argparse.ArgumentParser( description="Launch zgrab against IPs using port 22, 25, 443, or 465.") parser.add_argument( "-p", choices=["22", "25", "443", "465"], metavar="port", help="The port to scan: 22, 25, 443, or 465", ) parser.add_argument("-t", default=5, type=int, metavar="threadCount", help="The number of threads") parser.add_argument( "--mx", action="store_true", help="Scan only IPs from MX records. Useful for SMTP scans.", ) parser.add_argument( "-s", default=0, type=int, metavar="sleepTime", help="Sleep time in order to spread out the batches", ) parser.add_argument( "--qs", default=0, type=int, metavar="queueSize", help="How many hosts to scan in a batch", ) parser.add_argument("--zones_only", action="store_true", help="Scan only IPs from IP zones.") parser.add_argument( "--zgrab_path", default=global_zgrab_path, metavar="zgrabVersion", help="The version of ZGrab to use", ) args = parser.parse_args() if args.p == None: logger.error("A port value (22, 25, 443, or 465) must be provided.") exit(1) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() logger.warning(str(now) + ": I am already running! Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") rm_connector = RemoteMongoConnector.RemoteMongoConnector() all_dns_collection = rm_connector.get_all_dns_connection() ip_manager = IPManager.IPManager(rm_connector, True) jobs_manager = JobsManager.JobsManager(rm_connector, "zgrab_port_ip-" + args.p) jobs_manager.record_job_start() zones_struct = {} zones_struct["zones"] = ZoneManager.get_distinct_zones(rm_connector) # Not pretty but works zones_struct["ip_manager"] = ip_manager if args.mx: (ips, ip_context) = get_mx_ips(zones_struct["zones"], ip_manager, all_dns_collection) elif args.zones_only: (ips, ip_context) = get_only_ipzones(ip_manager.Tracked_CIDRs) else: (ips, ip_context) = get_ips(ip_manager, all_dns_collection) if args.s and int(args.s) > 0: global_sleep_time = int(args.s) if args.qs and int(args.qs) > 0: global_queue_size = int(args.qs) logger.info("Got IPs: " + str(len(ips))) zones_struct["ip_context"] = ip_context zgrab_collection = rm_connector.get_zgrab_port_data_connection() if args.p == "443": run_command = run_port_443_command elif args.p == "22": run_command = run_port_22_command elif args.p == "25": run_command = run_port_25_command elif args.p == "465": run_command = run_port_465_command check_save_location("./json_p" + args.p) global_zgrab_path = args.zgrab_path threads = [] logger.debug("Creating " + str(args.t) + " threads") for thread_id in range(1, args.t + 1): thread = ZgrabThread( thread_id, global_work_queue, args.p, run_command, zones_struct, zgrab_collection, ) thread.start() threads.append(thread) thread_id += 1 logger.info("Populating Queue") global_queue_lock.acquire() for ip in ips: global_work_queue.put(ip) global_queue_lock.release() # Wait for queue to empty while not global_work_queue.empty(): pass # Notify threads it's time to exit global_exit_flag = 1 # Wait for all threads to complete for t in threads: t.join() logger.info("Exiting Main Thread") logger.info("Global retest list: " + str(len(global_retest_list))) # Retest any SMTP hosts that did not respond to the StartTLS handshake if args.p == "25" and len(global_retest_list) > 0: process_thread( logger, global_retest_list, args.p, run_port_25_no_tls_command, zones_struct, zgrab_collection, "retest", ) # Remove old entries from before the scan if args.p == "443": other_results = zgrab_collection.find({ "data.tls": { "$exists": True }, "data.tls.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.tls": "" }}) elif args.p == "22": if "zgrab2" in global_zgrab_path: other_results = zgrab_collection.find({ "data.ssh": { "$exists": True }, "data.ssh.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.ssh": "" }}) else: other_results = zgrab_collection.find({ "data.xssh": { "$exists": True }, "data.xssh.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.xssh": "" }}) elif args.p == "25": other_results = zgrab_collection.find({ "data.smtp": { "$exists": True }, "data.smtp.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.smtp": "" }}) elif args.p == "465": other_results = zgrab_collection.find({ "data.smtps": { "$exists": True }, "data.smtps.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.smtps": "" }}) # Remove any completely empty entries zgrab_collection.delete_many({"data": {}}) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) if is_running(os.path.basename(__file__)): logger.warning("Already running...") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") r7 = Rapid7.Rapid7() mongo_connection = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connection) ip_manager = IPManager.IPManager(mongo_connection) rdns_collection = mongo_connection.get_sonar_reverse_dns_connection() zones = ZoneManager.get_distinct_zones(mongo_connection) logger.info("Zone length: " + str(len(zones))) save_directory = "./files/" parser = argparse.ArgumentParser( description='Parse Sonar files based on CIDRs.') parser.add_argument('--sonar_file_type', required=True, help='Specify "dns" or "rdns"') args = parser.parse_args() check_save_location(save_directory) # A session is necessary for the multi-step log-in process s = requests.Session() if args.sonar_file_type == "rdns": jobs_manager = JobsManager.JobsManager(mongo_connection, 'get_data_by_cidr_rdns') jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "rdns", jobs_manager) if html_parser.rdns_url == "": logger.error("Unknown Error") jobs_manager.record_job_error() exit(0) unzipped_rdns = download_remote_files(logger, s, html_parser.rdns_url, save_directory, jobs_manager) update_rdns(logger, unzipped_rdns, rdns_collection, dns_manager, ip_manager, zones) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) logger.info("RDNS Complete") jobs_manager.record_job_complete() elif args.sonar_file_type == "dns": jobs_manager = JobsManager.JobsManager(mongo_connection, 'get_data_by_cidr_dns') jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.any_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.any_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, dns_manager, ip_manager, zones) if html_parser.a_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.a_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, dns_manager, ip_manager, zones) if html_parser.aaaa_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.aaaa_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, dns_manager, ip_manager, zones) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) logger.info("DNS Complete") jobs_manager.record_job_complete() else: logger.error("Unrecognized sonar_file_type option. Exiting...") now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")