def main(): """ Begin Main... """ logger = LoggingUtil.create_log(__name__) now = datetime.now() print("Starting: " + str(now)) logger.info("Starting...") # Obtain the list of known email addresses from the config collection mongo_connector = MongoConnector.MongoConnector() whois_collection = mongo_connector.get_whois_connection() all_dns_collection = mongo_connector.get_all_dns_connection() zones_collection = mongo_connector.get_zone_connection() jobs_manager = JobsManager.JobsManager(mongo_connector, 'mark_expired') jobs_manager.record_job_start() # Grab all zones that are not expired of false_positives # Also exclude any that were recently created since they won't have data yet date_delta = datetime.today() - timedelta(days=30) zones = zones_collection.distinct( 'zone', { 'created': { "$lt": date_delta }, 'status': { "$nin": [ZoneManager.EXPIRED, ZoneManager.FALSE_POSITIVE] } }) # The Python Whois library is hit and miss with some international zones. # For now, this script focuses on the most popular TLDs. new_zones = get_primary_zones(logger, zones) expired_list = [] for zone in new_zones: if whois_collection.find({'zone': zone}).count() == 0: # Assume it is expired if there is no longer a whois record present expired_list.append(zone) for zone in expired_list: if all_dns_collection.find({'zone': zone}).count() > 0: # This may be a case where the Python Whois library failed # and the zone is still active. logger.debug("DNS records still exist for " + zone) expired_list.remove(zone) zone_manager = ZoneManager(mongo_connector) # Need to get this list before setting zones to expired in order to avoid a recursion problem. already_expired = zone_manager.get_zones_by_status(ZoneManager.EXPIRED) possibly_renewed = [] for zone in already_expired: if whois_collection.find({'zone': zone}).count() == 1: possibly_renewed.append(zone) for zone in expired_list: logger.debug("Expiring: " + zone) zone_manager.set_status(zone, ZoneManager.EXPIRED, "mark_expired.py") # Get the list of known registering entities. # This will only work for some whois lookups since Python Whois doesn't get # a valid org for all lookups and some have privacy enabled. config_collection = mongo_connector.get_config_connection() result = config_collection.find({}, { 'Whois_Orgs': 1, 'Whois_Name_Servers': 1 }) orgs = result[0]['Whois_Orgs'] name_servers = [] if 'Whois_Name_Servers' in result[0]: name_servers = result[0]['Whois_Name_Servers'] logger.debug(str(name_servers)) for zone in possibly_renewed: # We need to be careful of automatically marking something renewed # since it could have been registered by someone else. if whois_collection.find({ 'zone': zone, 'org': { "$in": orgs } }).count() == 1: logger.warning("ATTENTION: " + zone + " has been renewed based on org") zone_manager.set_status(zone, ZoneManager.UNCONFIRMED, "mark_expired.py") else: result = whois_collection.find({'zone': zone}, { 'name_servers': 1, "_id": 0 }) found = 0 if result is not None and 'name_servers' in result[0] and result[ 0]['name_servers'] is not None: for entry in result[0]['name_servers']: if entry.lower() in name_servers: logger.warning( "ATTENTION: " + zone + " has been renewed based on name servers") zone_manager.set_status(zone, ZoneManager.UNCONFIRMED, "mark_expired.py") found = 1 break if found == 0: result = whois_collection.find({'zone': zone}, { 'name_server_groups': 1, "_id": 0 }) if result is not None and 'name_server_groups' in result[ 0] and result[0]['name_server_groups'] is not None: for entry in result[0]['name_server_groups']: if entry.lower() in name_servers: logger.warning( "ATTENTION: " + zone + " has been renewed based on name server_groups" ) zone_manager.set_status(zone, ZoneManager.UNCONFIRMED, "mark_expired.py") found = 1 break if found == 0: logger.warning(zone + " has been renewed by an unknown entity") # Record status jobs_manager.record_job_complete() now = datetime.now() print("Ending: " + str(now)) logger.info("Complete.")
def main(): """ Begin Main... """ now = datetime.now() print("Starting: " + str(now)) # Obtain the list of known email addresses from the config collection mongo_connector = MongoConnector.MongoConnector() whois_collection = mongo_connector.get_whois_connection() all_dns_collection = mongo_connector.get_all_dns_connection() zones_collection = mongo_connector.get_zone_connection() jobs_collection = mongo_connector.get_jobs_connection() # Grab all zones that are not expired of false_positives # Also exclude any that were recently created since they won't have data yet date_delta = datetime.today() - timedelta(days=30) zones = zones_collection.distinct('zone', {'created': {"$lt": date_delta}, 'status': {"$nin": [ZoneManager.EXPIRED, ZoneManager.FALSE_POSITIVE]}}) # The Python Whois library is hit and miss with some international zones. # For now, this script focuses on the most popular TLDs. new_zones = get_primary_zones(zones) expired_list = [] for zone in new_zones: if whois_collection.find({'zone': zone}).count() == 0: # Assume it is expired if there is no longer a whois record present expired_list.append(zone) for zone in expired_list: if all_dns_collection.find({'zone': zone}).count() > 0: # This may be a case where the Python Whois library failed # and the zone is still active. print("DNS records still exist for " + zone) expired_list.remove(zone) zone_manager = ZoneManager(mongo_connector) # Need to get this list before setting zones to expired in order to avoid a recursion problem. already_expired = zone_manager.get_zones_by_status(ZoneManager.EXPIRED) possibly_renewed = [] for zone in already_expired: if whois_collection.find({'zone': zone}).count() == 1: possibly_renewed.append(zone) for zone in expired_list: print("Expiring: " + zone) zone_manager.set_status(zone, ZoneManager.EXPIRED, "mark_expired.py") # Get the list of known registering entities. # This will only work for some whois lookups since Python Whois doesn't get # a valid org for all lookups and some have privacy enabled. config_collection = mongo_connector.get_config_connection() result = config_collection.find({}, {'Whois_Orgs': 1}) orgs = result[0]['Whois_Orgs'] for zone in possibly_renewed: # We need to be careful of automatically marking something renewed # since it could have been registered by someone else. if whois_collection.find({'zone': zone, 'org': {"$in": orgs}}).count() == 1: print("ATTENTION: " + zone + "has been renewed") zone_manager.set_status(zone, ZoneManager.UNCONFIRMED, "mark_expired.py") else: print("WARNING: " + zone + " has been renewed by an unknown entity") # Record status jobs_collection.update_one({'job_name': 'mark_expired'}, {'$currentDate': {"updated": True}, "$set": {'status': 'COMPLETE'}}) now = datetime.now() print("Ending: " + str(now))