def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = MongoConnector.MongoConnector() remote_mongo_connector = RemoteMongoConnector.RemoteMongoConnector() jobs_collection = mongo_connector.get_jobs_connection() zone_list = update_zones(mongo_connector, remote_mongo_connector) update_ip_zones(mongo_connector, remote_mongo_connector) update_aws_cidrs(mongo_connector, remote_mongo_connector) update_azure_cidrs(mongo_connector, remote_mongo_connector) update_config(mongo_connector, remote_mongo_connector) update_braas(mongo_connector, remote_mongo_connector) update_all_dns(mongo_connector, remote_mongo_connector, zone_list) # Record status jobs_collection.update_one({'job_name': 'send_remote_server'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) now = datetime.now() print("Complete: " + str(now))
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) mongo_connector = RemoteMongoConnector.RemoteMongoConnector() jobs_collection = mongo_connector.get_jobs_connection() # Collect the tracked zones... zones = get_zones(mongo_connector) whois_collection = mongo_connector.get_whois_connection() for zone in zones: # Ensure the zone contains at least one dot. This is left over from an old bug. if zone.find(".") > 0: print(zone) zone_result = whois_collection.find_one({'zone': zone}) # If we haven't done a lookup in the past, try to collect the data. # A limit exists on the number of whois lookups you can perform so limit to new domains. if zone_result is None: do_whois_lookup(zone, whois_collection) # The cap on the number of old entries to be updated. MAX_OLD_ENTRIES = 400 # Grab entries that haven't been updated in 3 months last_week = datetime.now() - timedelta(days=90, hours=1) zone_result = whois_collection.find({ "updated": { "$lte": last_week } }).batch_size(10) i = 0 for result in zone_result: do_whois_lookup(result["zone"], whois_collection) i = i + 1 # Chances are that a lot of the entries were inserted on the same day. # This helps break updating old entries across different runs. if i > MAX_OLD_ENTRIES: break # Record status jobs_collection.update_one({'job_name': 'whois_lookups'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) now = datetime.now() print("Ending: " + str(now))
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") mongo_connector = RemoteMongoConnector.RemoteMongoConnector() jobs_manager = JobsManager.JobsManager(mongo_connector, 'whois_lookups') jobs_manager.record_job_start() # Collect the tracked zones... zones = get_zones(mongo_connector) whois_collection = mongo_connector.get_whois_connection() for zone in zones: # Ensure the zone contains at least one dot. This is left over from an old bug. if zone.find(".") > 0: logger.debug(zone) zone_result = whois_collection.find_one({'zone': zone}) # If we haven't done a lookup in the past, try to collect the data. # A limit exists on the number of whois lookups you can perform so limit to new domains. if zone_result is None: do_whois_lookup(logger, zone, whois_collection) # The cap on the number of old entries to be updated. MAX_OLD_ENTRIES = 400 # Grab entries that haven't been updated in 3 months last_week = datetime.now() - timedelta(days=90, hours=1) zone_result = whois_collection.find({ "updated": { "$lte": last_week } }).batch_size(10) i = 0 for result in zone_result: do_whois_lookup(logger, result["zone"], whois_collection) i = i + 1 # Chances are that a lot of the entries were inserted on the same day. # This helps break updating old entries across different runs. if i > MAX_OLD_ENTRIES: break # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): global global_exit_flag global global_retest_list global global_sleep_time global global_queue_size global_retest_list = [] parser = argparse.ArgumentParser(description='Launch zgrab against IPs using port 22, 25, 443, or 465.') parser.add_argument('-p', choices=['443','22', '25', '465'], metavar="port", help='The port to scan: 22, 25, 443, or 465') parser.add_argument('-t', default=5, type=int, metavar="threadCount", help='The number of threads') parser.add_argument('--mx', action="store_true", help='Scan only IPs from MX records. Useful for SMTP scans.') parser.add_argument('-s', default=0, type=int, metavar="sleepTime", help='Sleep time in order to spread out the batches') parser.add_argument('--qs', default=0, type=int, metavar="queueSize", help='How many hosts to scan in a batch') parser.add_argument('--zones_only', action="store_true", help='Scan only IPs from IP zones.') args = parser.parse_args() if args.p == None: print("A port value (22, 25, 443, or 465) must be provided.") exit(0) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() print(str(now) + ": I am already running! Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) rm_connector = RemoteMongoConnector.RemoteMongoConnector() all_dns_collection = rm_connector.get_all_dns_connection() zones_struct = {} zones_struct['zones'] = ZoneManager.get_distinct_zones(rm_connector) zones_struct['ip_zones'] = get_ip_zones(rm_connector) # Collect the list of AWS CIDRs zones_struct['aws_ips'] = get_aws_ips(rm_connector) # Collect the list of Azure CIDRs zones_struct['azure_ips'] = get_azure_ips(rm_connector) if args.mx: (ips, ip_context) = get_mx_ips(zones_struct['zones'], all_dns_collection) elif args.zones_only: (ips, ip_context) = get_only_ipzones(zones_struct['ip_zones']) else: (ips, ip_context) = get_ips(zones_struct['ip_zones'], all_dns_collection) if args.s and int(args.s) > 0: global_sleep_time = int(args.s) if args.qs and int(args.qs) > 0: global_queue_size = int(args.qs) print("Got IPs: " + str(len(ips))) zones_struct['ip_context'] = ip_context zgrab_collection = rm_connector.get_zgrab_port_data_connection() if args.p == "443": run_command = run_port_443_command elif args.p == "22": run_command = run_port_22_command elif args.p == "25": run_command = run_port_25_command elif args.p == "465": run_command = run_port_465_command threads = [] print ("Creating " + str(args.t) + " threads") for thread_id in range (1, args.t + 1): thread = ZgrabThread(thread_id, global_work_queue, args.p, run_command, zones_struct, zgrab_collection) thread.start() threads.append(thread) thread_id += 1 print("Populating Queue") global_queue_lock.acquire() for ip in ips: global_work_queue.put(ip) global_queue_lock.release() # Wait for queue to empty while not global_work_queue.empty(): pass # Notify threads it's time to exit global_exit_flag = 1 # Wait for all threads to complete for t in threads: t.join() print ("Exiting Main Thread") print("Global retest list: " + str(len(global_retest_list))) # Retest any SMTP hosts that did not respond to the StartTLS handshake if args.p == "25" and len(global_retest_list) > 0: process_thread(global_retest_list, args.p, run_port_25_no_tls_command, zones_struct, zgrab_collection, "retest") # Remove old entries from before the scan if args.p == "443": other_results = zgrab_collection.find({'data.tls': {"$exists": True}, 'data.tls.timestamp': {"$lt": now}}) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.tls': ""}}) elif args.p == "22": other_results = zgrab_collection.find({'data.xssh': {"$exists": True}, 'data.xssh.timestamp': {"$lt": now}}) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.xssh': ""}}) elif args.p == "25": other_results = zgrab_collection.find({'data.smtp': {"$exists": True}, 'data.smtp.timestamp': {"$lt": now}}) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.smtp': ""}}) elif args.p == "465": other_results = zgrab_collection.find({'data.smtps': {"$exists": True}, 'data.smtps.timestamp': {"$lt": now}}) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.smtps': ""}}) # Remove any completely empty entries zgrab_collection.remove({'data': {}}) now = datetime.now() print("Complete: " + str(now))
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") parser = argparse.ArgumentParser( description= 'Send specific collections to the remote MongoDB. If no arguments are provided, then all data is mirrored.' ) parser.add_argument('--send_zones', action="store_true", required=False, help='Send IP zones') parser.add_argument('--send_ip_zones', action="store_true", required=False, help='Send IP zones') parser.add_argument('--send_third_party_zones', action="store_true", required=False, help='Send AWS, Azure, etc.') parser.add_argument('--send_config', action="store_true", required=False, help='Send configs') parser.add_argument('--send_dns_records', action="store_true", required=False, help='Send DNS records') args = parser.parse_args() send_all = False if len(sys.argv) == 1: send_all = True mongo_connector = MongoConnector.MongoConnector() remote_mongo_connector = RemoteMongoConnector.RemoteMongoConnector() jobs_manager = JobsManager.JobsManager(mongo_connector, 'send_remote_server') jobs_manager.record_job_start() if send_all or args.send_zones: zone_list = update_zones(logger, mongo_connector, remote_mongo_connector, True) else: zone_list = update_zones(logger, mongo_connector, remote_mongo_connector, False) if send_all or args.send_ip_zones: update_ip_zones(logger, mongo_connector, remote_mongo_connector) if send_all or args.send_third_party_zones: update_aws_cidrs(logger, mongo_connector, remote_mongo_connector) update_azure_cidrs(logger, mongo_connector, remote_mongo_connector) update_akamai_cidrs(logger, mongo_connector, remote_mongo_connector) update_gcp_cidrs(logger, mongo_connector, remote_mongo_connector) if send_all or args.send_config: update_config(logger, mongo_connector, remote_mongo_connector) if send_all or args.send_dns_records: update_all_dns(logger, mongo_connector, remote_mongo_connector, zone_list) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") parser = argparse.ArgumentParser( description="Send specific collections to the remote MongoDB. If no arguments are provided, then all data is mirrored." ) parser.add_argument( "--send_zones", action="store_true", required=False, help="Send IP zones" ) parser.add_argument( "--send_ip_zones", action="store_true", required=False, help="Send IP zones" ) parser.add_argument( "--send_third_party_zones", action="store_true", required=False, help="Send AWS, Azure, etc.", ) parser.add_argument( "--send_config", action="store_true", required=False, help="Send configs" ) parser.add_argument( "--send_dns_records", action="store_true", required=False, help="Replace all DNS records", ) parser.add_argument( "--send_dns_diff", action="store_true", required=False, help="Send new DNS records", ) parser.add_argument( "--date_diff", default=2, type=int, help="The number of days used for identifying new records in send_dns_diff", ) args = parser.parse_args() send_all = False if len(sys.argv) == 1: send_all = True mongo_connector = MongoConnector.MongoConnector() remote_mongo_connector = RemoteMongoConnector.RemoteMongoConnector() jobs_manager = JobsManager.JobsManager(mongo_connector, "send_remote_server") jobs_manager.record_job_start() if send_all or args.send_zones: try: zone_list = update_zones( logger, mongo_connector, remote_mongo_connector, True ) except: logger.error( "Could not communicate with the remote database when sending zones" ) jobs_manager.record_job_error() exit(1) else: zone_list = update_zones(logger, mongo_connector, remote_mongo_connector, False) if send_all or args.send_ip_zones: try: update_ip_zones(logger, mongo_connector, remote_mongo_connector) except: logger.error( "Could not communicate with the remote database when sending IP zones" ) jobs_manager.record_job_error() exit(1) if send_all or args.send_third_party_zones: try: update_aws_cidrs(logger, mongo_connector, remote_mongo_connector) update_azure_cidrs(logger, mongo_connector, remote_mongo_connector) update_akamai_cidrs(logger, mongo_connector, remote_mongo_connector) update_gcp_cidrs(logger, mongo_connector, remote_mongo_connector) except: logger.error( "Could not communicate with the remote database when sending third-party zones" ) jobs_manager.record_job_error() exit(1) if send_all or args.send_config: try: update_config(logger, mongo_connector, remote_mongo_connector) except: logger.error( "Could not communicate with the remote database when sending config data" ) jobs_manager.record_job_error() exit(1) # This will completely repopulate the DNS records table. if args.send_dns_records is not False: try: update_all_dns(logger, mongo_connector, remote_mongo_connector, zone_list) except: logger.error( "Could not communicate with the remote database when sending DNS records" ) jobs_manager.record_job_error() exit(1) # If you have a large data set, then you may only want to send updated records if send_all or args.send_dns_diff: try: update_all_dns_diff_mode( logger, mongo_connector, remote_mongo_connector, zone_list, args.date_diff, ) except: logger.error( "Could not communicate with the remote database when sending DNS diff records" ) jobs_manager.record_job_error() exit(1) # Record status jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin main... """ logger = LoggingUtil.create_log(__name__) if is_running("get_censys_files.py"): """ Check to see if a download is in process... """ logger.warning("Can't run due to get_files running. Goodbye!") exit(0) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ logger.warning("I am already running! Goodbye!") exit(0) # Make the relevant database connections RMC = RemoteMongoConnector.RemoteMongoConnector() ip_manager = IPManager.IPManager(RMC) # Verify that the get_files script has a recent file in need of parsing. jobs_collection = RMC.get_jobs_connection() status = jobs_collection.find_one({"job_name": "censys"}) if status["status"] != "DOWNLOADED": logger.warning("The status is not set to DOWNLOADED. Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Collect the list of available zones zones = ZoneManager.get_distinct_zones(RMC) logger.info("Zones: " + str(len(zones))) # Get the current configuration information for Marinus. config_collection = RMC.get_config_connection() configs = config_collection.find({}) orgs = [] for org in configs[0]["SSL_Orgs"]: orgs.append(org) logger.info("Orgs: " + str(len(orgs))) # Obtain the name of the decompressed file. filename_f = open(FILENAME_FILE, "r") decompressed_file = filename_f.readline() filename_f.close() # For manual testing: decompressed_file = "ipv4.json" logger.info("Beginning file processing...") # Remove old results from the database results_collection = RMC.get_results_connection() results_collection.delete_many({}) all_dns_collection = RMC.get_all_dns_connection() try: with open(decompressed_file, "r") as dec_f: for line in dec_f: try: entry = json.loads(line) """ Does the SSL certificate match a known organization? Is the IP address in a known CIDR? Is the IP address recorded in Splunk? """ if (check_in_org(entry, orgs) or ip_manager.is_tracked_ip(entry["ip"]) or ip_manager.find_splunk_data(entry["ip"], "AWS") is not None or ip_manager.find_splunk_data( entry["ip"], "AZURE") is not None): entry["zones"] = check_in_zone(entry, zones) entry["aws"] = ip_manager.is_aws_ip(entry["ip"]) entry["azure"] = ip_manager.is_azure_ip(entry["ip"]) (domains, zones) = lookup_domain(entry, zones, all_dns_collection) if len(domains) > 0: entry["domains"] = domains if len(zones) > 0: for zone in zones: if zone not in entry["zones"]: entry["zones"].append(zone) insert_result(entry, results_collection) # else: # #This will add days to the amount of time necessary to scan the file. # matched_zones = check_in_zone(entry, zones) # if matched_zones != []: # entry['zones'] = matched_zones # entry['aws'] = ip_manager.is_aws_ip(entry['ip']) # entry['azure'] = ip_manager.is_azure_ip(entry['ip']) # insert_result(entry, results_collection) except ValueError as err: logger.error("Value Error!") logger.error(str(err)) except: logger.error("Line unexpected error: " + str(sys.exc_info()[0])) logger.error("Line unexpected error: " + str(sys.exc_info()[1])) except IOError as err: logger.error("I/O error({0}): {1}".format(err.errno, err.strerror)) exit(1) except: logger.error("Unexpected error: " + str(sys.exc_info()[0])) logger.error("Unexpected error: " + str(sys.exc_info()[1])) exit(1) # Indicate that the processing of the job is complete and ready for download to Marinus jobs_collection.update_one( {"job_name": "censys"}, { "$currentDate": { "updated": True }, "$set": { "status": "COMPLETE" } }, ) now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin main... """ if is_running("get_censys_files.py"): """ Check to see if a download is in process... """ now = datetime.now() print(str(now) + ": Can't run due to get_files running. Goodbye!") exit(0) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() print(str(now) + ": I am already running! Goodbye!") exit(0) # Make the relevant database connections RMC = RemoteMongoConnector.RemoteMongoConnector() # Verify that the get_files script has a recent file in need of parsing. jobs_collection = RMC.get_jobs_connection() status = jobs_collection.find_one({'job_name': 'censys'}) if status['status'] != "DOWNLOADED": now = datetime.now() print(str(now) + ": The status is not set to DOWNLOADED. Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) # Collect the list of available zones zones = ZoneManager.get_distinct_zones(RMC) print("Zones: " + str(len(zones))) # Collect the list of AWS CIDRs aws_ips = [] get_aws_ips(RMC, aws_ips) print("AWS IPs: " + str(len(aws_ips))) # Collect the list of Azure CIDRs azure_ips = [] get_azure_ips(RMC, azure_ips) print("Azure IPs: " + str(len(azure_ips))) # Collect the list of known CIDRs ip_zones_collection = RMC.get_ipzone_connection() results = ip_zones_collection.find({'status': {"$ne": "false_positive"}}) cidrs = [] for entry in results: cidrs.append(IPNetwork(entry['zone'])) print("CIDRs: " + str(len(cidrs))) # Get the current configuration information for Marinus. config_collection = RMC.get_config_connection() configs = config_collection.find({}) orgs = [] for org in configs[0]['SSL_Orgs']: orgs.append(org) print("Orgs: " + str(len(orgs))) # Obtain the name of the decompressed file. filename_f = open(FILENAME_FILE, "r") decompressed_file = filename_f.readline() filename_f.close() # For manual testing: decompressed_file = "ipv4.json" now = datetime.now() print(str(now) + ": Beginning file processing...") # Remove old results from the database results_collection = RMC.get_results_connection() results_collection.remove({}) all_dns_collection = RMC.get_all_dns_connection() try: with open(decompressed_file, "r") as dec_f: for line in dec_f: try: entry = json.loads(line) """ Does the SSL certificate match a known organization? Is the IP address in a known CIDR? """ if check_in_org(entry, orgs) or \ check_in_cidr(entry['ip'], cidrs): entry['zones'] = check_in_zone(entry, zones) entry['aws'] = is_aws_ip(entry['ip'], aws_ips) entry['azure'] = is_azure_ip(entry['ip'], azure_ips) (domains, zones) = lookup_domain(entry, zones, all_dns_collection) if len(domains) > 0: entry['domains'] = domains if len(zones) > 0: for zone in zones: if zone not in entry['zones']: entry['zones'].append(zone) insert_result(entry, results_collection) # else: # #This will add days to the amount of time necessary to scan the file. # matched_zones = check_in_zone(entry, zones) # if matched_zones != []: # entry['zones'] = matched_zones # entry['aws'] = is_aws_ip(entry['ip'], aws_ips) # entry['azure'] = is_azure_ip(entry['ip'], azure_ips) # insert_result(entry, results_collection) except ValueError as err: print("Value Error!") print(str(err)) except: print("Line unexpected error:", sys.exc_info()[0]) print("Line unexpected error:", sys.exc_info()[1]) except IOError as err: print("I/O error({0}): {1}".format(err.errno, err.strerror)) exit(0) except: print("Unexpected error:", sys.exc_info()[0]) print("Unexpected error:", sys.exc_info()[1]) exit(0) # Indicate that the processing of the job is complete and ready for download to Marinus jobs_collection.update_one({'job_name': 'censys'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'COMPLETE' } }) now = datetime.now() print("Ending: " + str(now))
def main(): """ Begin Main... """ global global_exit_flag global global_zgrab_path logger = LoggingUtil.create_log(__name__) parser = argparse.ArgumentParser( description="Launch zgrab against IPs using port 80 or 443.") parser.add_argument("-p", choices=["443", "80"], metavar="port", help="The web port: 80 or 443") parser.add_argument("-t", default=5, type=int, metavar="threadCount", help="The number of threads") parser.add_argument( "--zgrab_path", default=global_zgrab_path, metavar="zgrabVersion", help="The version of ZGrab to use", ) args = parser.parse_args() if args.p == None: logger.error("A port value (80 or 443) must be provided.") exit(1) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() logger.warning(str(now) + ": I am already running! Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") rm_connector = RemoteMongoConnector.RemoteMongoConnector() all_dns_collection = rm_connector.get_all_dns_connection() ip_manager = IPManager.IPManager(rm_connector, True) jobs_manager = JobsManager.JobsManager(rm_connector, "zgrab_http_ip-" + args.p) jobs_manager.record_job_start() zones_struct = {} zones_struct["zones"] = ZoneManager.get_distinct_zones(rm_connector) # Not pretty but cleaner than previous method zones_struct["ip_manager"] = ip_manager (ips, ip_context) = get_ips(ip_manager, all_dns_collection) logger.info("Got IPs: " + str(len(ips))) zones_struct["ip_context"] = ip_context if args.p == "443": zgrab_collection = rm_connector.get_zgrab_443_data_connection() run_command = run_port_443_command else: zgrab_collection = rm_connector.get_zgrab_80_data_connection() run_command = run_port_80_command check_save_location("./json_p" + args.p) global_zgrab_path = args.zgrab_path threads = [] logger.debug("Creating " + str(args.t) + " threads") for thread_id in range(1, args.t + 1): thread = ZgrabThread( thread_id, global_work_queue, args.p, run_command, zones_struct, zgrab_collection, ) thread.start() threads.append(thread) thread_id += 1 logger.info("Populating Queue") global_queue_lock.acquire() for ip in ips: global_work_queue.put(ip) global_queue_lock.release() # Wait for queue to empty while not global_work_queue.empty(): pass # Notify threads it's time to exit global_exit_flag = 1 # Wait for all threads to complete for t in threads: t.join() logger.info("Exiting Main Thread") # Remove last week's old entries lastweek = datetime.now() - timedelta(days=7) zgrab_collection.delete_many({ "ip": { "$ne": "<nil>" }, "timestamp": { "$lt": lastweek } }) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Connect to the remote databases mongo_connector = MongoConnector.MongoConnector() rm_connector = RemoteMongoConnector.RemoteMongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zones = ZoneManager.get_distinct_zones(mongo_connector) jobs_manager = JobsManager.JobsManager(mongo_connector, "remote_download") jobs_manager.record_job_start() remote_jobs_collection = rm_connector.get_jobs_connection() # Check the status of the Censys job on the remote database try: status = remote_jobs_collection.find_one({"job_name": "censys"}) except: logger.error("Can not connect to remote database") jobs_manager.record_job_error() exit(1) if (status is not None and "status" in status and status["status"] != jobs_manager.COMPLETE): logger.info("Censys scans status is not COMPLETE") elif (status is not None and "status" in status and status["status"] == jobs_manager.COMPLETE): # Get connections to the relevant collections. censys_collection = mongo_connector.get_zgrab_443_data_connection() remote_censys_collection = rm_connector.get_zgrab_443_data_connection() download_censys_scan_info(logger, censys_collection, remote_censys_collection) # Tell the remote database that is safe to start processing the next Censys file remote_jobs_collection.update_one( {"job_name": "censys"}, { "$currentDate": { "updated": True }, "$set": { "status": jobs_manager.READY } }, ) # Get connections to the relevant HTTPS collections. zgrab_443_data_collection = mongo_connector.get_zgrab_443_data_connection() remote_zgrab_443_data_collection = rm_connector.get_zgrab_443_data_connection( ) download_zgrab_info(logger, zgrab_443_data_collection, remote_zgrab_443_data_collection) # Get connections to the relevant HTTP collections. zgrab_80_data_collection = mongo_connector.get_zgrab_80_data_connection() remote_zgrab_80_data_collection = rm_connector.get_zgrab_80_data_connection( ) download_zgrab_info(logger, zgrab_80_data_collection, remote_zgrab_80_data_collection) # Get connections to the relevant port collections. zgrab_port_data_collection = mongo_connector.get_zgrab_port_data_connection( ) remote_zgrab_port_data_collection = rm_connector.get_zgrab_port_data_connection( ) download_zgrab_port_info(logger, zgrab_port_data_collection, remote_zgrab_port_data_collection) # Download latest whois information status = remote_jobs_collection.find_one({"job_name": "whois_lookups"}) if status["status"] == jobs_manager.COMPLETE: whois_collection = mongo_connector.get_whois_connection() remote_whois_collection = rm_connector.get_whois_connection() download_whois_data(logger, whois_collection, remote_whois_collection) remote_jobs_collection.update_one( {"job_name": "whois"}, {"$set": { "status": jobs_manager.READY }}) # Download Amass results amass_collection = mongo_connector.get_owasp_amass_connection() remote_amass_collection = rm_connector.get_owasp_amass_connection() download_amass_data(logger, amass_collection, remote_amass_collection, dns_manager, zones) # Download the status of the remote jobs download_jobs_status(logger, jobs_manager._jobs_collection, remote_jobs_collection) # Download remote sonar DNS findings download_sonar_dns(logger, dns_manager, rm_connector) # Download remote sonar RDNS findings download_sonar_rdns(logger, mongo_connector, rm_connector) # Update the local jobs database to done jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ global global_exit_flag global global_zgrab_path logger = LoggingUtil.create_log(__name__) parser = argparse.ArgumentParser( description='Launch zgrab against domains using port 80 or 443.') parser.add_argument('-p', choices=['443', '80'], metavar="port", help='The web port: 80 or 443') parser.add_argument('-t', default=5, type=int, metavar="threadCount", help='The number of threads') parser.add_argument('--zgrab_path', default=global_zgrab_path, metavar='zgrabVersion', help='The version of ZGrab to use') args = parser.parse_args() if args.p == None: logger.error("A port value (80 or 443) must be provided.") exit(1) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() logger.warning(str(now) + ": I am already running! Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") rm_connector = RemoteMongoConnector.RemoteMongoConnector() all_dns_collection = rm_connector.get_all_dns_connection() jobs_manager = JobsManager.JobsManager(rm_connector, "zgrab_http_domain-" + args.p) jobs_manager.record_job_start() if args.p == "443": zgrab_collection = rm_connector.get_zgrab_443_data_connection() run_command = run_port_443_command else: zgrab_collection = rm_connector.get_zgrab_80_data_connection() run_command = run_port_80_command check_save_location("./json_p" + args.p) global_zgrab_path = args.zgrab_path zones = ZoneManager.get_distinct_zones(rm_connector) ip_manager = IPManager.IPManager(rm_connector) for zone in zones: global_exit_flag = 0 domains = get_domains(all_dns_collection, ip_manager, zone) if len(domains) == 0: continue num_threads = args.t if len(domains) < args.t: num_threads = len(domains) logger.debug("Creating " + str(num_threads) + " threads") threads = [] for thread_id in range(1, num_threads + 1): thread = ZgrabThread(thread_id, global_work_queue, args.p, run_command, zone, zgrab_collection) thread.start() threads.append(thread) thread_id += 1 logger.debug(zone + " length: " + str(len(domains))) logger.info("Populating Queue") global_queue_lock.acquire() for domain in domains: global_work_queue.put(domain) global_queue_lock.release() # Wait for queue to empty while not global_work_queue.empty(): pass logger.info("Queue empty") # Notify threads it's time to exit global_exit_flag = 1 # Wait for all threads to complete for t in threads: t.join() # Remove last week's old entries lastweek = datetime.now() - timedelta(days=7) zgrab_collection.remove({ 'domain': { "$ne": "<nil>" }, 'timestamp': { "$lt": lastweek } }) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Begin main... """ # Don't run if the search files script is working on the existing file. if is_running("search_censys_files_new.py"): now = datetime.now() print("File search running: " + str(now)) exit(0) # Record the start in the logs now = datetime.now() print("Starting: " + str(now)) RMC = RemoteMongoConnector.RemoteMongoConnector() jobs_collection = RMC.get_jobs_connection() # Obtain the timestamp of the last file that was downloaded. last_timestamp = "0" try: f_time = open(TIMESTAMP_FILE, "r") last_timestamp = f_time.readline() f_time.close() except FileNotFoundError: last_timestamp = "0" # Get the meta data for the currently available file. req = requests.get(CENSYS_API + "data/ipv4", auth=HTTPBasicAuth(CENSYS_APP_ID, CENSYS_SECRET)) if req.status_code != 200: print("Error " + str(req.status_code) + ": Unable to query Censys Data API\n") print(req.text) time.sleep(60) req = requests.get(CENSYS_API + "data/ipv4", auth=HTTPBasicAuth(CENSYS_APP_ID, CENSYS_SECRET)) if req.status_code != 200: print("Error on IPv4 retry. Giving up...") exit(0) data_json = json.loads(req.text) # Get the timestamp for the currently available file timestamp = data_json['results']['latest']['timestamp'] # If it is the same file as last time, then don't download again. if last_timestamp == timestamp: print("Already downloaded. Exiting...") exit(0) else: print("Old timestamp: " + last_timestamp) print("New timestamp: " + timestamp) # Get the location of the details for the new file details_url = data_json['results']['latest']['details_url'] req = requests.get(details_url, auth=HTTPBasicAuth(CENSYS_APP_ID, CENSYS_SECRET)) if req.status_code != 200: print("Error " + str(req.status_code) + ": Unable to query Censys Details API\n") print(req.text) time.sleep(60) req = requests.get(details_url, auth=HTTPBasicAuth(CENSYS_APP_ID, CENSYS_SECRET)) if req.status_code != 200: print("Error on details retry. Giving up...") exit(0) data_json = json.loads(req.text) compressed_path = data_json['primary_file']['compressed_download_path'] print(compressed_path) # Record the timestamp of the file that we are about to download. time_f = open(TIMESTAMP_FILE, "w") time_f.write(timestamp) time_f.close() # Remove any old files. subprocess.call(["rm", DECOMPRESSED_FILE]) subprocess.call("rm *.lz4", shell=True) # Download the new file. filename = download_file(compressed_path) # Decompress the new file into the file indicated by "DECOMPRESSED_FILE" subprocess.check_call(["lz4", "-d", filename, DECOMPRESSED_FILE]) # Record the name of the filename that has the output dec_f = open(FILENAME_FILE, "w") dec_f.write(DECOMPRESSED_FILE) dec_f.close() # Record that we successfully downloaded the file and that search files can start. jobs_collection.update_one({'job_name': 'censys'}, { '$currentDate': { "updated": True }, "$set": { 'status': 'DOWNLOADED' } }) now = datetime.now() print("Complete: " + str(now))
def main(): """ Begin main... """ logger = LoggingUtil.create_log(__name__) mongo_connector = RemoteMongoConnector.RemoteMongoConnector() now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") jobs_manager = JobsManager.JobsManager(mongo_connector, "owasp_amass") amass_collection = mongo_connector.get_owasp_amass_connection() output_dir = "./amass_files/" arg_parser = argparse.ArgumentParser( description= "Run the OWASP Amass tool and store the results in the database.") arg_parser.add_argument( "--config_file", required=False, help="An optional Amass config file. Otherwise, defaults will be used.", ) arg_parser.add_argument("--amass_path", required=True, help="The path to the amass binary") arg_parser.add_argument( "--output_dir", default=output_dir, help="The local path where to save Amass files.", ) arg_parser.add_argument( "--docker_output_dir", default=output_dir, help="The path within Docker where to save Amass files.", ) arg_parser.add_argument( "--amass_version", type=int, default=3, help="The version of OWASP Amass being used.", ) arg_parser.add_argument( "--amass_mode", required=False, type=str, default="local", choices=["local", "docker"], help="The version of OWASP Amass being used.", ) arg_parser.add_argument( "--amass_timeout", required=False, type=str, help="The timeout value for the Amass command line", ) arg_parser.add_argument( "--exclude_zones", required=False, type=str, default="", help="A comma delimited list of sub-strings used to exclude zones", ) arg_parser.add_argument( "--exclude_regex", required=False, type=str, default="", help="Exclude a list of domains containing a substring", ) arg_parser.add_argument( "--created_within_last", required=False, type=int, default=0, help="Only process zones created within the last x days", ) arg_parser.add_argument( "--if_list", required=False, type=str, default="", help= "The amass -if list of sources to include. Can't be used with -ef.", ) arg_parser.add_argument( "--ef_list", required=False, type=str, default="", help= "The amass -ef list of sources to exclude. Can't be used with -if.", ) arg_parser.add_argument( "--sleep", type=int, default=5, help= "Sleep time in seconds between amass runs so as not to overuse service limits.", ) args = arg_parser.parse_args() if args.amass_mode == "local" and not os.path.isfile(args.amass_path): logger.error("Incorrect amass_path argument provided") exit(1) # In Docker mode, this would be relative to the Docker path and not the system path if (args.amass_mode == "local" and "config_file" in args and not os.path.isfile(args.config_file)): logger.error("Incorrect config_file location") exit(1) if "output_dir" in args: output_dir = args.output_dir if not output_dir.endswith("/"): output_dir = output_dir + "/" # In Docker mode, this would be relative to the Docker path and not the system path if args.amass_mode == "local": check_save_location(output_dir) jobs_manager.record_job_start() if args.created_within_last > 0: zone_collection = mongo_connector.get_zone_connection() past_create_date = datetime.now() - timedelta( days=args.created_within_last) results = mongo_connector.perform_find( zone_collection, {"created": { "$gt": past_create_date }}) zones = [] for entry in results: zones.append(entry["zone"]) elif args.exclude_regex is not None and len(args.exclude_regex) > 0: exclude_re = re.compile(".*" + args.exclude_regex + ".*") zone_collection = mongo_connector.get_zone_connection() results = mongo_connector.perform_find( zone_collection, { "$and": [ { "zone": { "$not": exclude_re } }, { "status": { "$nin": [ZoneManager.FALSE_POSITIVE, ZoneManager.EXPIRED] } }, ] }, ) zones = [] for entry in results: zones.append(entry["zone"]) else: zones = ZoneManager.get_distinct_zones(mongo_connector) # If the job died half way through, you can skip over domains that were already processed # when you restart the script. new_zones = [] for zone in zones: if not os.path.isfile(output_dir + zone + "-do.json"): new_zones.append(zone) exclude_strings = args.exclude_zones.split() # If exclude_strings was specified, then remove any matching zones if len(exclude_strings) > 0: for zone in new_zones: for entry in exclude_strings: if entry in zone: new_zones.remove(zone) # Recently updated zones # This helps reduce the number of redundant scans if you stop and restart all_dns_collection = mongo_connector.get_all_dns_connection() scrub_date = datetime.now() - timedelta(days=120, hours=9) recent_zones = mongo_connector.perform_distinct( all_dns_collection, "zone", { "sources.source": { "$regex": "amass:.*" }, "sources.updated": { "$gt": scrub_date }, }, ) for zone in recent_zones: if zone in new_zones: new_zones.remove(zone) logger.info("New Zones Length: " + str(len(new_zones))) for zone in new_zones: # Pace out calls to the Amass services time.sleep(args.sleep) if args.amass_mode == "local": command_line = [] command_line.append(args.amass_path) else: command_line = args.amass_path.split() if int(args.amass_version) >= 3: command_line.append("enum") if args.config_file: command_line.append("-config") command_line.append(args.config_file) if args.amass_timeout: command_line.append("-timeout") command_line.append(args.amass_timeout) if args.if_list: command_line.append("-if") command_line.append(args.if_list) if args.ef_list: command_line.append("-ef") command_line.append(args.ef_list) command_line.append("-d") command_line.append(zone) command_line.append("-src") command_line.append("-ip") command_line.append("-nolocaldb") command_line.append("-json") command_line.append(args.docker_output_dir + zone + "-do.json") try: subprocess.check_call(command_line) except subprocess.CalledProcessError as e: # Even when there is an error, there will likely still be results. # We can continue with the data that was collected thus far. logger.warning("ERROR: Amass run exited with a non-zero status: " + str(e)) if os.path.isfile(output_dir + zone + "-do.json"): output = open(output_dir + zone + "-do.json", "r") json_data = [] for line in output: try: json_data.append(json.loads(line)) except: logger.warning("Amass wrote an incomplete line: " + str(line)) output.close() for finding in json_data: finding["timestamp"] = datetime.now() """ Results from amass squash the cname records and only provides the final IPs. Therefore, we have to re-do the DNS lookup in download_from_remote_database. This collection is just a recording of the original results """ mongo_connector.perform_insert(amass_collection, finding) # Clear old findings amass_collection.delete_many({"timestamp": {"$lt": now}}) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
""" This script downloads updates from the remote MongoDB server that is used for larger jobs. This script is only necessary if a remote MongoDB is set up. This script can be run daily. """ import logging from datetime import datetime, timedelta from libs3 import MongoConnector, RemoteMongoConnector, JobsManager from libs3.LoggingUtil import LoggingUtil # Connect to the remote databases mongo_connector = MongoConnector.MongoConnector() rm_connector = RemoteMongoConnector.RemoteMongoConnector() def download_censys_scan_info(censys_collection, remote_censys_collection): """ Download the latest censys scan information """ # Grab the new results from the remote server. results = remote_censys_collection.find({}, {"_id": 0}) # Remove the previous results from the local Censys collection censys_collection.remove({}) # Insert the new results from the remote server into the local server for result in results: censys_collection.insert(result)
def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) if is_running(os.path.basename(__file__)): logger.warning("Already running...") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") parser = argparse.ArgumentParser( description="Parse Sonar files based on domain zones.") parser.add_argument( "--sonar_file_type", choices=["dns-any", "dns-a", "rdns"], required=True, help='Specify "dns-any", "dns-a", or "rdns"', ) parser.add_argument( "--database", choices=["local", "remote"], required=False, default="local", help="Whether to use the local or remote DB", ) args = parser.parse_args() r7 = Rapid7.Rapid7() if args.database == "remote": mongo_connector = RemoteMongoConnector.RemoteMongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector, "get_sonar_data_dns") else: mongo_connector = MongoConnector.MongoConnector() dns_manager = DNSManager.DNSManager(mongo_connector) zones = ZoneManager.get_distinct_zones(mongo_connector) save_directory = "./files/" check_save_location(save_directory) # A session is necessary for the multi-step log-in process s = requests.Session() if args.sonar_file_type == "rdns": logger.info("Updating RDNS records") jobs_manager = JobsManager.JobsManager(mongo_connector, "get_sonar_data_rdns") jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "rdns", jobs_manager) if html_parser.rdns_url == "": logger.error("Unknown Error") jobs_manager.record_job_error() exit(0) unzipped_rdns = download_remote_files(logger, s, html_parser.rdns_url, save_directory, jobs_manager) update_rdns(logger, unzipped_rdns, zones, dns_manager, mongo_connector) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) jobs_manager.record_job_complete() elif args.sonar_file_type == "dns-any": logger.info("Updating DNS ANY records") jobs_manager = JobsManager.JobsManager(mongo_connector, "get_sonar_data_dns-any") jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.any_url != "": unzipped_dns = download_remote_files(logger, s, html_parser.any_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, dns_manager) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) jobs_manager.record_job_complete() elif args.sonar_file_type == "dns-a": logger.info("Updating DNS A, AAAA, and CNAME records") jobs_manager = JobsManager.JobsManager(mongo_connector, "get_sonar_data_dns-a") jobs_manager.record_job_start() try: html_parser = r7.find_file_locations(s, "fdns", jobs_manager) if html_parser.a_url != "": logger.info("Updating A records") unzipped_dns = download_remote_files(logger, s, html_parser.a_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, dns_manager) if html_parser.aaaa_url != "": logger.info("Updating AAAA records") unzipped_dns = download_remote_files(logger, s, html_parser.aaaa_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, dns_manager) if html_parser.cname_url != "": logger.info("Updating CNAME records") unzipped_dns = download_remote_files(logger, s, html_parser.cname_url, save_directory, jobs_manager) update_dns(logger, unzipped_dns, zones, dns_manager) except Exception as ex: logger.error("Unexpected error: " + str(ex)) jobs_manager.record_job_error() exit(0) jobs_manager.record_job_complete() else: logger.error("Unrecognized sonar_file_type option. Exiting...") now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): """ Beging Main... """ global global_exit_flag global global_retest_list global global_sleep_time global global_queue_size global global_zgrab_path logger = LoggingUtil.create_log(__name__) global_retest_list = [] parser = argparse.ArgumentParser( description="Launch zgrab against IPs using port 22, 25, 443, or 465.") parser.add_argument( "-p", choices=["22", "25", "443", "465"], metavar="port", help="The port to scan: 22, 25, 443, or 465", ) parser.add_argument("-t", default=5, type=int, metavar="threadCount", help="The number of threads") parser.add_argument( "--mx", action="store_true", help="Scan only IPs from MX records. Useful for SMTP scans.", ) parser.add_argument( "-s", default=0, type=int, metavar="sleepTime", help="Sleep time in order to spread out the batches", ) parser.add_argument( "--qs", default=0, type=int, metavar="queueSize", help="How many hosts to scan in a batch", ) parser.add_argument("--zones_only", action="store_true", help="Scan only IPs from IP zones.") parser.add_argument( "--zgrab_path", default=global_zgrab_path, metavar="zgrabVersion", help="The version of ZGrab to use", ) args = parser.parse_args() if args.p == None: logger.error("A port value (22, 25, 443, or 465) must be provided.") exit(1) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() logger.warning(str(now) + ": I am already running! Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") rm_connector = RemoteMongoConnector.RemoteMongoConnector() all_dns_collection = rm_connector.get_all_dns_connection() ip_manager = IPManager.IPManager(rm_connector, True) jobs_manager = JobsManager.JobsManager(rm_connector, "zgrab_port_ip-" + args.p) jobs_manager.record_job_start() zones_struct = {} zones_struct["zones"] = ZoneManager.get_distinct_zones(rm_connector) # Not pretty but works zones_struct["ip_manager"] = ip_manager if args.mx: (ips, ip_context) = get_mx_ips(zones_struct["zones"], ip_manager, all_dns_collection) elif args.zones_only: (ips, ip_context) = get_only_ipzones(ip_manager.Tracked_CIDRs) else: (ips, ip_context) = get_ips(ip_manager, all_dns_collection) if args.s and int(args.s) > 0: global_sleep_time = int(args.s) if args.qs and int(args.qs) > 0: global_queue_size = int(args.qs) logger.info("Got IPs: " + str(len(ips))) zones_struct["ip_context"] = ip_context zgrab_collection = rm_connector.get_zgrab_port_data_connection() if args.p == "443": run_command = run_port_443_command elif args.p == "22": run_command = run_port_22_command elif args.p == "25": run_command = run_port_25_command elif args.p == "465": run_command = run_port_465_command check_save_location("./json_p" + args.p) global_zgrab_path = args.zgrab_path threads = [] logger.debug("Creating " + str(args.t) + " threads") for thread_id in range(1, args.t + 1): thread = ZgrabThread( thread_id, global_work_queue, args.p, run_command, zones_struct, zgrab_collection, ) thread.start() threads.append(thread) thread_id += 1 logger.info("Populating Queue") global_queue_lock.acquire() for ip in ips: global_work_queue.put(ip) global_queue_lock.release() # Wait for queue to empty while not global_work_queue.empty(): pass # Notify threads it's time to exit global_exit_flag = 1 # Wait for all threads to complete for t in threads: t.join() logger.info("Exiting Main Thread") logger.info("Global retest list: " + str(len(global_retest_list))) # Retest any SMTP hosts that did not respond to the StartTLS handshake if args.p == "25" and len(global_retest_list) > 0: process_thread( logger, global_retest_list, args.p, run_port_25_no_tls_command, zones_struct, zgrab_collection, "retest", ) # Remove old entries from before the scan if args.p == "443": other_results = zgrab_collection.find({ "data.tls": { "$exists": True }, "data.tls.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.tls": "" }}) elif args.p == "22": if "zgrab2" in global_zgrab_path: other_results = zgrab_collection.find({ "data.ssh": { "$exists": True }, "data.ssh.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.ssh": "" }}) else: other_results = zgrab_collection.find({ "data.xssh": { "$exists": True }, "data.xssh.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.xssh": "" }}) elif args.p == "25": other_results = zgrab_collection.find({ "data.smtp": { "$exists": True }, "data.smtp.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.smtp": "" }}) elif args.p == "465": other_results = zgrab_collection.find({ "data.smtps": { "$exists": True }, "data.smtps.timestamp": { "$lt": now } }) for result in other_results: zgrab_collection.update_one({"_id": ObjectId(result["_id"])}, {"$unset": { "data.smtps": "" }}) # Remove any completely empty entries zgrab_collection.delete_many({"data": {}}) jobs_manager.record_job_complete() now = datetime.now() print("Complete: " + str(now)) logger.info("Complete.")
def main(): global global_exit_flag parser = argparse.ArgumentParser( description='Launch zgrab against IPs using port 80 or 443.') parser.add_argument('-p', choices=['443', '80'], metavar="port", help='The web port: 80 or 443') parser.add_argument('-t', default=5, type=int, metavar="threadCount", help='The number of threads') args = parser.parse_args() if args.p == None: print("A port value (80 or 443) must be provided.") exit(0) if is_running(os.path.basename(__file__)): """ Check to see if a previous attempt to parse is still running... """ now = datetime.now() print(str(now) + ": I am already running! Goodbye!") exit(0) now = datetime.now() print("Starting: " + str(now)) rm_connector = RemoteMongoConnector.RemoteMongoConnector() all_dns_collection = rm_connector.get_all_dns_connection() zones_struct = {} zones_struct['zones'] = ZoneManager.get_distinct_zones(rm_connector) zones_struct['ip_zones'] = get_ip_zones(rm_connector) # Collect the list of AWS CIDRs zones_struct['aws_ips'] = get_aws_ips(rm_connector) # Collect the list of Azure CIDRs zones_struct['azure_ips'] = get_azure_ips(rm_connector) (ips, ip_context) = get_ips(zones_struct['ip_zones'], all_dns_collection) print("Got IPs: " + str(len(ips))) zones_struct['ip_context'] = ip_context if args.p == "443": zgrab_collection = rm_connector.get_zgrab_443_data_connection() run_command = run_port_443_command else: zgrab_collection = rm_connector.get_zgrab_80_data_connection() run_command = run_port_80_command threads = [] print("Creating " + str(args.t) + " threads") for thread_id in range(1, args.t + 1): thread = ZgrabThread(thread_id, global_work_queue, args.p, run_command, zones_struct, zgrab_collection) thread.start() threads.append(thread) thread_id += 1 print("Populating Queue") global_queue_lock.acquire() for ip in ips: global_work_queue.put(ip) global_queue_lock.release() # Wait for queue to empty while not global_work_queue.empty(): pass # Notify threads it's time to exit global_exit_flag = 1 # Wait for all threads to complete for t in threads: t.join() print("Exiting Main Thread") # Remove last week's old entries lastweek = datetime.now() - timedelta(days=7) zgrab_collection.remove({ 'ip': { "$ne": "<nil>" }, 'timestamp': { "$lt": lastweek } }) now = datetime.now() print("Complete: " + str(now))