def extract_rdns_info(self, ip):
"""
Extract RDNS domain and zone information from the IP address
"""
rnds_value = self.find_reverse_dns(ip)
if rnds_value is None:
return "", None
if self._ZONES is None:
self._ZONES = ZoneManager.get_distinct_zones(self.mongo_connector)
rdns_zone = ZoneManager.get_root_domain(rnds_value, None)
return rnds_value, rdns_zone
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
# Make database connections
mc_connector = MongoConnector.MongoConnector()
file_path = "/mnt/workspace/ct_facebook/"
fb_connector = FacebookConnector.FacebookConnector()
access_token = fb_connector.get_facebook_access_token()
zones = ZoneManager.get_distinct_zones(mc_connector)
for zone in zones:
time.sleep(15)
results = fetch_domain(fb_connector, access_token, zone)
if results is None:
print("ERROR looking up: " + zone)
continue
print(zone + ": " + str(len(results)))
for result in results:
cert_f = open(file_path + zone + "_" + result['id'] + ".pem", "w")
cert_f.write(result['certificate_pem'])
cert_f.close()
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print ("Starting: " + str(now))
logger.info("Starting...")
dns_types = {"a":1, "ns":2, "cname":5, "soa":6, "ptr":12, "hinfo": 13, "mx": 15, "txt":16, "aaaa":28, "srv":33, "naptr": 35, "ds": 43, "rrsig": 46, "dnskey": 48}
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector, 'marinus_dns')
jobs_manager.record_job_start()
dns_manager = DNSManager.DNSManager(mongo_connector)
zones = ZoneManager.get_distinct_zones(mongo_connector)
google_dns = GoogleDNS.GoogleDNS()
for zone in zones:
time.sleep(1)
for dtype, dnum in dns_types.items():
result = google_dns.fetch_DNS_records(zone, dnum)
if result == []:
logger.debug("No records found for " + zone)
else:
new_record = result[0]
new_record['status'] = 'confirmed'
new_record['zone'] = zone
new_record['created'] = datetime.now()
logger.debug ("Found " + dtype + " for: " + zone)
dns_manager.insert_record(new_record, "marinus")
logger.info("Starting SOA Search")
soa_searches = find_sub_zones(all_dns_collection)
for entry in soa_searches:
time.sleep(1)
result = google_dns.fetch_DNS_records(zone, dns_types['soa'])
if result != []:
new_record = result[0]
new_record['status'] = 'confirmed'
new_record['zone'] = get_fld_from_value(entry, '')
new_record['created'] = datetime.now()
logger.debug ("Found SOA: " + entry)
if new_record['zone'] != '':
dns_manager.insert_record(new_record, "marinus")
jobs_manager.record_job_complete()
now = datetime.now()
print ("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_collection = mongo_connector.get_jobs_connection()
zone_ingestor = ZoneIngestor.ZoneIngestor()
current_zones = ZoneManager.get_distinct_zones(mongo_connector)
# For cases with multiple R53 accounts, include the account id for reference
sts = boto3.client('sts')
account_id = sts.get_caller_identity()["Arn"].split(':')[4]
r53_source = "R53:" + str(account_id)
r53_client = boto3.client('route53')
r53_domains = r53_client.list_hosted_zones()
r53_zone_list = []
while r53_domains != {}:
for zone_data in r53_domains['HostedZones']:
# Only add public zones
if zone_data['Config']['PrivateZone'] == False:
r53_zone_list.append(zone_data)
if r53_domains['IsTruncated'] == True:
r53_domains = r53_client.list_domains(
Marker=r53_domains['NextMarker'])
else:
r53_domains = {}
for zone_data in r53_zone_list:
# Double check that this is not a new zone
zone_name = zone_data['Name'][:-1]
if zone_name not in current_zones:
print("Creating zone: " + zone_name)
zone_ingestor.add_zone(zone_data['Name'], r53_source)
# Add hosts to the zone
update_records(r53_client, dns_manager, zone_data, r53_source)
# Record status
jobs_collection.update_one({'job_name': 'get_route53'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zone_ingestor = ZoneIngestor.ZoneIngestor()
jobs_manager = JobsManager.JobsManager(mongo_connector, "get_route53")
jobs_manager.record_job_start()
current_zones = ZoneManager.get_distinct_zones(mongo_connector)
# For cases with multiple R53 accounts, include the account id for reference
sts = boto3.client("sts")
account_id = sts.get_caller_identity()["Arn"].split(":")[4]
r53_source = "R53:" + str(account_id)
r53_client = boto3.client("route53")
r53_domains = r53_client.list_hosted_zones()
r53_zone_list = []
while r53_domains != {}:
for zone_data in r53_domains["HostedZones"]:
# Only add public zones
if zone_data["Config"]["PrivateZone"] == False:
r53_zone_list.append(zone_data)
if r53_domains["IsTruncated"] == True:
r53_domains = r53_client.list_domains(
Marker=r53_domains["NextMarker"])
else:
r53_domains = {}
for zone_data in r53_zone_list:
# Double check that this is not a new zone
zone_name = zone_data["Name"][:-1]
if zone_name not in current_zones:
logger.info("Creating zone: " + zone_name)
zone_ingestor.add_zone(zone_data["Name"], r53_source)
# Add hosts to the zone
update_records(r53_client, dns_manager, zone_data, r53_source)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
# Create an instance of the VirusTotal class
vt_instance = VirusTotal.VirusTotal()
# Get collections for the queries
mongo_connector = MongoConnector.MongoConnector()
vt_collection = mongo_connector.get_virustotal_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector,
'get_virustotal_data')
jobs_manager.record_job_start()
# Collect the list of tracked TLDs
zones = ZoneManager.get_distinct_zones(mongo_connector)
# For each tracked TLD
for zone in zones:
print("Checking " + zone)
results = vt_instance.get_domain_report(zone)
if results is None:
print("Error querying zone " + zone)
elif results['response_code'] == -1:
print("VT unhappy with " + zone)
elif results['response_code'] == 0:
print("VT doesn't have " + zone)
else:
print("Matched " + zone)
results['zone'] = zone
results['created'] = datetime.now()
# Mongo doesn't allow key names with periods in them
# Re-assign to an undotted key name
if "Dr.Web category" in results:
results['Dr Web category'] = results.pop("Dr.Web category")
vt_collection.delete_one({"zone": zone})
vt_collection.insert(results)
# This sleep command is so that we don't exceed the daily limit on the free API
# This setting results in this script taking several days to complete
time.sleep(25)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
def __ultradns_zone_info_response_handler(self, response):
"""
Handles the API response. Incorrect JSON parsing is allowed upto 20 times post which the
script exits.
:param response: Response object
"""
try:
response_data = response.json()
record_sets = response_data["rrSets"]
except (ValueError, AttributeError) as err:
if self.UH.incorrect_response_json_allowed > 0:
self._logger.warning(
"Unable to parse response JSON for zone " +
self.zone_queried)
self.UH.incorrect_response_json_allowed -= 1
else:
self.APIH.handle_api_error(
"Unable to parse response JSON for 20 zones: " + repr(err),
self.UH.jobs_manager,
)
else:
for record in record_sets:
dns_info = dict()
# The ownerName could be either the FQDN or a relative domain name.
# In case it is a FQDN it will end in '.'
fqdn = record["ownerName"] + "." + self.zone_queried
if record["ownerName"].endswith("."):
fqdn = record["ownerName"][:-1]
# A get_root_domain lookup is performed because UDNS supports sub-zones
dns_info["zone"] = ZoneManager.get_root_domain(
self.zone_queried)
dns_info["fqdn"] = fqdn
dns_info["type"] = record["rrtype"].split(" ")[0].lower()
dns_info["status"] = "unknown"
for dns in record["rdata"]:
if dns_info["type"] in ["a", "ptr"]:
try:
if IPAddress(dns).is_private():
continue
except AddrFormatError as err:
self._logger.warning("For " + fqdn +
" encountered: " + str(err))
continue
if not (dns_info["type"] in ["soa", "txt"
]) and dns.endswith("."):
dns = dns[:-1]
dns_info["value"] = dns
dns_info["created"] = datetime.now()
self.DNS_MGR.insert_record(dns_info.copy(), self.UH.source)
self.UH.set_offset(response_data["resultInfo"])
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
"get_external_cnames")
jobs_manager.record_job_start()
groups = {}
# Collect zones
zone_results = ZoneManager.get_distinct_zones(mongo_connector)
zones = []
for zone in zone_results:
if zone.find(".") >= 0:
zones.append(zone)
# Collect the all_dns cnames.
logger.info("Starting All DNS...")
all_dns_recs = dns_manager.find_multiple({"type": "cname"}, None)
for srec in all_dns_recs:
if not is_tracked_zone(srec["value"], zones):
add_to_list(
get_fld_from_value(srec["value"], srec["zone"]),
srec["fqdn"],
srec["value"],
srec["zone"],
groups,
)
# Update the database
tpds_collection = mongo_connector.get_tpds_connection()
tpds_collection.delete_many({})
for key in groups.keys():
tpds_collection.insert_one(groups[key])
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
now = datetime.now()
print ("Starting: " + str(now))
dns_types = {"a":1, "ns":2, "cname":5, "soa":6, "ptr":12, "hinfo": 13, "mx": 15, "txt":16, "aaaa":28, "srv":33, "naptr": 35, "ds": 43, "rrsig": 46, "dnskey": 48}
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
jobs_collection = mongo_connector.get_jobs_connection()
dns_manager = DNSManager.DNSManager(mongo_connector)
zones = ZoneManager.get_distinct_zones(mongo_connector)
google_dns = GoogleDNS.GoogleDNS()
for zone in zones:
time.sleep(1)
for dtype, dnum in dns_types.items():
result = google_dns.fetch_DNS_records(zone, dnum)
if result == []:
print("No records found for " + zone)
else:
new_record = result[0]
new_record['status'] = 'confirmed'
new_record['zone'] = zone
new_record['created'] = datetime.now()
print ("Found " + dtype + " for: " + zone)
dns_manager.insert_record(new_record, "marinus")
print("Starting SOA Search")
soa_searches = find_sub_zones(all_dns_collection)
for entry in soa_searches:
time.sleep(1)
result = google_dns.fetch_DNS_records(zone, dns_types['soa'])
if result != []:
new_record = result[0]
new_record['status'] = 'confirmed'
new_record['zone'] = get_fld_from_value(entry, '')
new_record['created'] = datetime.now()
print ("Found SOA: " + entry)
if new_record['zone'] != '':
dns_manager.insert_record(new_record, "marinus")
jobs_collection.update_one({'job_name': 'marinus_dns'},
{'$currentDate': {"updated" : True},
"$set": {'status': 'COMPLETE'}})
now = datetime.now()
print ("Complete: " + str(now))
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_collection = mongo_connector.get_jobs_connection()
groups = {}
# Collect zones
zone_results = ZoneManager.get_distinct_zones(mongo_connector)
zones = []
for zone in zone_results:
if zone.find(".") >= 0:
zones.append(zone)
# Collect the all_dns cnames.
print("Starting All DNS...")
all_dns_recs = dns_manager.find_multiple({'type': 'cname'}, None)
for srec in all_dns_recs:
if not is_tracked_zone(srec['value'], zones):
add_to_list(get_fld_from_value(srec['value'], srec['zone']),
srec['fqdn'], srec['value'], srec['zone'], groups)
# Update the database
tpds_collection = mongo_connector.get_tpds_connection()
tpds_collection.remove({})
for key in groups.keys():
tpds_collection.insert(groups[key])
# Record status
jobs_collection.update_one({'job_name': 'get_external_cnames'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Ending: " + str(now))
def get_infoblox_extattr(self):
"""
Extracts the zones from the zone collection to query Infoblox. The API calls continue to be made
for the zone till the next_page_id is set to None indicating no new results to be fetched.
Post the retrieval of all the data, the archaic data for a zone and record_type is purged.
"""
zones = ZoneManager.get_zones_by_source(self.MC, 'Infoblox')
for zone in zones:
self.zone_queried = zone
self.next_page_id = None
self.__get_previous_records()
self.__infoblox_paginated_request()
while self.next_page_id:
self.__infoblox_paginated_request()
self.IH.clean_collection(self.previous_records, self.iblox_extattr_collection)
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
if is_running(os.path.basename(__file__)):
logger.warning("Already running...")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
r7 = Rapid7.Rapid7()
mongo_connection = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connection)
ip_manager = IPManager.IPManager(mongo_connection)
rdns_collection = mongo_connection.get_sonar_reverse_dns_connection()
zones = ZoneManager.get_distinct_zones(mongo_connection)
logger.info("Zone length: " + str(len(zones)))
save_directory = "./files/"
parser = argparse.ArgumentParser(
description='Parse Sonar files based on CIDRs.')
parser.add_argument('--sonar_file_type',
required=True,
help='Specify "dns" or "rdns"')
args = parser.parse_args()
check_save_location(save_directory)
# A session is necessary for the multi-step log-in process
s = requests.Session()
if args.sonar_file_type == "rdns":
jobs_manager = JobsManager.JobsManager(mongo_connection,
'get_data_by_cidr_rdns')
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "rdns", jobs_manager)
if html_parser.rdns_url == "":
logger.error("Unknown Error")
jobs_manager.record_job_error()
exit(0)
unzipped_rdns = download_remote_files(logger, s,
html_parser.rdns_url,
save_directory, jobs_manager)
update_rdns(logger, unzipped_rdns, rdns_collection, dns_manager,
ip_manager, zones)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
logger.info("RDNS Complete")
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns":
jobs_manager = JobsManager.JobsManager(mongo_connection,
'get_data_by_cidr_dns')
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.any_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.any_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, dns_manager, ip_manager,
zones)
if html_parser.a_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.a_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, dns_manager, ip_manager,
zones)
if html_parser.aaaa_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.aaaa_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, dns_manager, ip_manager,
zones)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
logger.info("DNS Complete")
jobs_manager.record_job_complete()
else:
logger.error("Unrecognized sonar_file_type option. Exiting...")
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
google_dns = GoogleDNS.GoogleDNS()
jobs_manager = JobsManager.JobsManager(mongo_connector,
'extract_ssl_domains')
jobs_manager.record_job_start()
parser = argparse.ArgumentParser(
description='Search TLS certificates for additional DNS names')
parser.add_argument('--zgrab_version',
default=2,
type=int,
choices=[1, 2],
metavar="version",
help='The version of ZGrab used to collect data')
args = parser.parse_args()
dns_names = []
round_two = []
zones = ZoneManager.get_distinct_zones(mongo_connector)
# Collect the list of domains from the SSL Certificates
extract_ct_certificate_names(dns_names, mongo_connector)
# extract_censys_certificate_names(dns_names, mongo_connector)
if args.zgrab_version == 1:
extract_zgrab_certificate_names(logger, dns_names, mongo_connector)
else:
extract_zgrab2_certificate_names(logger, dns_names, mongo_connector)
input_list = []
# Some SSL certificates are for multiple domains.
# The tracked company may not own all domains.
# Therefore, we filter to only the root domains that belong to the tracked company.
logger.info("Pre-filter list: " + str(len(dns_names)))
for hostname in dns_names:
if not hostname.startswith("*"):
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
# Pause to prevent DoS-ing of Google's HTTPS DNS Service
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
if ip_addr['type'] == "cname" and is_tracked_zone(
ip_addr['value'], zones):
add_to_round_two(ip_addr['value'], round_two)
else:
logger.warning("Failed IP Lookup for: " + hostname)
else:
logger.warning("Failed match on zone for: " + hostname)
else:
logger.warning("Skipping wildcard: " + hostname)
dead_dns_collection = mongo_connector.get_dead_dns_connection()
# Some DNS records will be CNAME records pointing to other tracked domains.
# This is a single level recursion to lookup those domains.
logger.info("Round Two list: " + str(len(round_two)))
for hostname in round_two:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
else:
logger.warning("Failed IP Lookup for: " + hostname)
original_record = dns_manager.find_one({"fqdn": hostname},
"ssl")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
else:
logger.warning("Failed match on zone for: " + hostname)
# Record all the results.
dns_manager.remove_by_source("ssl")
logger.info("List length: " + str(len(input_list)))
for final_result in input_list:
dns_manager.insert_record(final_result, "ssl")
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Obtain the list of known email addresses from the config collection
mongo_connector = MongoConnector.MongoConnector()
whois_collection = mongo_connector.get_whois_connection()
all_dns_collection = mongo_connector.get_all_dns_connection()
zones_collection = mongo_connector.get_zone_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector, 'mark_expired')
jobs_manager.record_job_start()
# Grab all zones that are not expired of false_positives
# Also exclude any that were recently created since they won't have data yet
date_delta = datetime.today() - timedelta(days=30)
zones = zones_collection.distinct(
'zone', {
'created': {
"$lt": date_delta
},
'status': {
"$nin": [ZoneManager.EXPIRED, ZoneManager.FALSE_POSITIVE]
}
})
# The Python Whois library is hit and miss with some international zones.
# For now, this script focuses on the most popular TLDs.
new_zones = get_primary_zones(logger, zones)
expired_list = []
for zone in new_zones:
if whois_collection.find({'zone': zone}).count() == 0:
# Assume it is expired if there is no longer a whois record present
expired_list.append(zone)
for zone in expired_list:
if all_dns_collection.find({'zone': zone}).count() > 0:
# This may be a case where the Python Whois library failed
# and the zone is still active.
logger.debug("DNS records still exist for " + zone)
expired_list.remove(zone)
zone_manager = ZoneManager(mongo_connector)
# Need to get this list before setting zones to expired in order to avoid a recursion problem.
already_expired = zone_manager.get_zones_by_status(ZoneManager.EXPIRED)
possibly_renewed = []
for zone in already_expired:
if whois_collection.find({'zone': zone}).count() == 1:
possibly_renewed.append(zone)
for zone in expired_list:
logger.debug("Expiring: " + zone)
zone_manager.set_status(zone, ZoneManager.EXPIRED, "mark_expired.py")
# Get the list of known registering entities.
# This will only work for some whois lookups since Python Whois doesn't get
# a valid org for all lookups and some have privacy enabled.
config_collection = mongo_connector.get_config_connection()
result = config_collection.find({}, {
'Whois_Orgs': 1,
'Whois_Name_Servers': 1
})
orgs = result[0]['Whois_Orgs']
name_servers = []
if 'Whois_Name_Servers' in result[0]:
name_servers = result[0]['Whois_Name_Servers']
logger.debug(str(name_servers))
for zone in possibly_renewed:
# We need to be careful of automatically marking something renewed
# since it could have been registered by someone else.
if whois_collection.find({
'zone': zone,
'org': {
"$in": orgs
}
}).count() == 1:
logger.warning("ATTENTION: " + zone +
" has been renewed based on org")
zone_manager.set_status(zone, ZoneManager.UNCONFIRMED,
"mark_expired.py")
else:
result = whois_collection.find({'zone': zone}, {
'name_servers': 1,
"_id": 0
})
found = 0
if result is not None and 'name_servers' in result[0] and result[
0]['name_servers'] is not None:
for entry in result[0]['name_servers']:
if entry.lower() in name_servers:
logger.warning(
"ATTENTION: " + zone +
" has been renewed based on name servers")
zone_manager.set_status(zone, ZoneManager.UNCONFIRMED,
"mark_expired.py")
found = 1
break
if found == 0:
result = whois_collection.find({'zone': zone}, {
'name_server_groups': 1,
"_id": 0
})
if result is not None and 'name_server_groups' in result[
0] and result[0]['name_server_groups'] is not None:
for entry in result[0]['name_server_groups']:
if entry.lower() in name_servers:
logger.warning(
"ATTENTION: " + zone +
" has been renewed based on name server_groups"
)
zone_manager.set_status(zone,
ZoneManager.UNCONFIRMED,
"mark_expired.py")
found = 1
break
if found == 0:
logger.warning(zone + " has been renewed by an unknown entity")
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Make database connections
mongo_connector = MongoConnector.MongoConnector()
ct_collection = mongo_connector.get_certificate_transparency_connection()
config_collection = mongo_connector.get_config_connection()
x509parser = X509Parser.X509Parser()
zones = ZoneManager.get_distinct_zones(mongo_connector)
result = config_collection.find_one({}, {'SSL_Orgs': 1, "_id": 0})
ssl_orgs = result['SSL_Orgs']
# Defaults
save_location = '/mnt/workspace/'
download_method = 'dbAndSave'
save_type = "PEM"
parser = argparse.ArgumentParser(
description='Download certificate information from the provide CT Log.'
)
parser.add_argument(
'--log_source',
required=True,
help=
'Indicates which log to query based on values in the x509Parser library'
)
parser.add_argument(
'--include_precerts',
action="store_true",
help='Include pre-certificates which are not finalized')
parser.add_argument(
'--download_methods',
choices=['dbAndSave', 'dbOnly'],
default=download_method,
help=
'Indicates whether to download the raw files or just save to the database'
)
parser.add_argument(
'--starting_index',
required=False,
default=-1,
type=int,
help='Force the script to start at specific index within the log.')
parser.add_argument(
'--cert_save_location',
required=False,
default=save_location,
help=
'Indicates where to save the certificates on disk when choosing dbAndSave'
)
parser.add_argument(
'--save_type',
choices=['PEM', 'ASN1'],
default=save_type,
help='Indicates which format to use for the data. The default is PEM')
args = parser.parse_args()
source = args.log_source
try:
ct_log_map = x509parser.CT_LOG_MAP[source]
except:
logger.error("ERROR: UNKNOWN LOG SOURCE: " + source)
exit(1)
if args.cert_save_location:
save_location = args.cert_save_location
if not save_location.endswith("/"):
save_location = save_location + "/"
if args.download_methods:
download_method = args.download_methods
check_save_location(save_location, source)
if args.save_type:
save_type = args.save_type
jobs_manager = JobsManager.JobsManager(mongo_connector, "ct_log-" + source)
jobs_manager.record_job_start()
if args.starting_index == -1:
starting_index = fetch_starting_index(ct_collection, source)
else:
starting_index = args.starting_index
logger.info("Starting Index: " + str(starting_index))
sth_data = fetch_sth(logger, "https://" + ct_log_map['url'])
logger.info("Tree size: " + str(sth_data['tree_size']))
current_index = starting_index
while current_index < sth_data['tree_size']:
ending_index = current_index + 256
if ending_index > sth_data['tree_size']:
ending_index = sth_data['tree_size']
logger.debug("Checking from index: " + str(current_index) +
" to index " + str(ending_index))
certs = fetch_certificate_batch(logger, "https://" + ct_log_map['url'],
current_index, ending_index)
for entry in certs['entries']:
der_cert, cert_type = get_cert_from_leaf(logger,
entry['leaf_input'])
if der_cert is None and cert_type == 1 and not args.include_precerts:
current_index = current_index + 1
continue
elif der_cert is None and cert_type == 0:
current_index = current_index + 1
continue
elif der_cert is None and cert_type == 1:
der_cert = get_cert_from_extra_data(entry['extra_data'])
cert = x509parser.parse_data(der_cert, source)
if cert is None:
logger.warning("Skipping certificate index: " +
str(current_index))
current_index = current_index + 1
continue
if cert_type == 1:
cert['ct_log_type'] = "PRE-CERTIFICATE"
else:
cert['ct_log_type'] = "CERTIFICATE"
cert_zones = check_zone_relevancy(cert, zones)
if check_org_relevancy(cert, ssl_orgs) or cert_zones != []:
cert[source + "_id"] = current_index
cert['zones'] = cert_zones
logger.info("Adding " + source + " id: " + str(current_index) +
" SHA256: " + cert['fingerprint_sha256'])
insert_certificate(cert, source, ct_collection, cert_zones)
if download_method == 'dbAndSave':
write_file(logger, cert, save_location, save_type, source)
current_index = current_index + 1
# Set isExpired for any entries that have recently expired.
ct_collection.update(
{
"not_after": {
"$lt": datetime.utcnow()
},
"isExpired": False
}, {"$set": {
"isExpired": True
}},
multi=True)
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
global global_exit_flag
global global_retest_list
global global_sleep_time
global global_queue_size
global_retest_list = []
parser = argparse.ArgumentParser(description='Launch zgrab against IPs using port 22, 25, 443, or 465.')
parser.add_argument('-p', choices=['443','22', '25', '465'], metavar="port", help='The port to scan: 22, 25, 443, or 465')
parser.add_argument('-t', default=5, type=int, metavar="threadCount", help='The number of threads')
parser.add_argument('--mx', action="store_true", help='Scan only IPs from MX records. Useful for SMTP scans.')
parser.add_argument('-s', default=0, type=int, metavar="sleepTime", help='Sleep time in order to spread out the batches')
parser.add_argument('--qs', default=0, type=int, metavar="queueSize", help='How many hosts to scan in a batch')
parser.add_argument('--zones_only', action="store_true", help='Scan only IPs from IP zones.')
args = parser.parse_args()
if args.p == None:
print("A port value (22, 25, 443, or 465) must be provided.")
exit(0)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
now = datetime.now()
print(str(now) + ": I am already running! Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
rm_connector = RemoteMongoConnector.RemoteMongoConnector()
all_dns_collection = rm_connector.get_all_dns_connection()
zones_struct = {}
zones_struct['zones'] = ZoneManager.get_distinct_zones(rm_connector)
zones_struct['ip_zones'] = get_ip_zones(rm_connector)
# Collect the list of AWS CIDRs
zones_struct['aws_ips'] = get_aws_ips(rm_connector)
# Collect the list of Azure CIDRs
zones_struct['azure_ips'] = get_azure_ips(rm_connector)
if args.mx:
(ips, ip_context) = get_mx_ips(zones_struct['zones'], all_dns_collection)
elif args.zones_only:
(ips, ip_context) = get_only_ipzones(zones_struct['ip_zones'])
else:
(ips, ip_context) = get_ips(zones_struct['ip_zones'], all_dns_collection)
if args.s and int(args.s) > 0:
global_sleep_time = int(args.s)
if args.qs and int(args.qs) > 0:
global_queue_size = int(args.qs)
print("Got IPs: " + str(len(ips)))
zones_struct['ip_context'] = ip_context
zgrab_collection = rm_connector.get_zgrab_port_data_connection()
if args.p == "443":
run_command = run_port_443_command
elif args.p == "22":
run_command = run_port_22_command
elif args.p == "25":
run_command = run_port_25_command
elif args.p == "465":
run_command = run_port_465_command
threads = []
print ("Creating " + str(args.t) + " threads")
for thread_id in range (1, args.t + 1):
thread = ZgrabThread(thread_id, global_work_queue, args.p, run_command, zones_struct, zgrab_collection)
thread.start()
threads.append(thread)
thread_id += 1
print("Populating Queue")
global_queue_lock.acquire()
for ip in ips:
global_work_queue.put(ip)
global_queue_lock.release()
# Wait for queue to empty
while not global_work_queue.empty():
pass
# Notify threads it's time to exit
global_exit_flag = 1
# Wait for all threads to complete
for t in threads:
t.join()
print ("Exiting Main Thread")
print("Global retest list: " + str(len(global_retest_list)))
# Retest any SMTP hosts that did not respond to the StartTLS handshake
if args.p == "25" and len(global_retest_list) > 0:
process_thread(global_retest_list, args.p, run_port_25_no_tls_command, zones_struct, zgrab_collection, "retest")
# Remove old entries from before the scan
if args.p == "443":
other_results = zgrab_collection.find({'data.tls': {"$exists": True}, 'data.tls.timestamp': {"$lt": now}})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.tls': ""}})
elif args.p == "22":
other_results = zgrab_collection.find({'data.xssh': {"$exists": True}, 'data.xssh.timestamp': {"$lt": now}})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.xssh': ""}})
elif args.p == "25":
other_results = zgrab_collection.find({'data.smtp': {"$exists": True}, 'data.smtp.timestamp': {"$lt": now}})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.smtp': ""}})
elif args.p == "465":
other_results = zgrab_collection.find({'data.smtps': {"$exists": True}, 'data.smtps.timestamp': {"$lt": now}})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.smtps': ""}})
# Remove any completely empty entries
zgrab_collection.remove({'data': {}})
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
'extract_mx_domains')
google_dns = GoogleDNS.GoogleDNS()
jobs_manager.record_job_start()
dns_names = []
round_two = []
zones = ZoneManager.get_distinct_zones(mongo_connector)
# Collect the list of domains from the MX Records
extract_mx_names(dns_names, dns_manager)
input_list = []
# Some MX records point to the third-party domains.
# Therefore, we filter to only the root domains that belong to the tracked company.
print("Pre-filter list: " + str(len(dns_names)))
for hostname in dns_names:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
# Pause to prevent DoS-ing of Google's HTTPS DNS Service
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
if ip_addr['type'] == "cname" and is_tracked_zone(
ip_addr['value'], zones):
add_to_round_two(ip_addr['value'], round_two)
else:
print("Failed IP Lookup for: " + hostname)
else:
print("Failed match on zone for: " + hostname)
dead_dns_collection = mongo_connector.get_dead_dns_connection()
# Some DNS records will be CNAME records pointing to other tracked domains.
# This is a single level recursion to lookup those domains.
print("Round Two list: " + str(len(round_two)))
for hostname in round_two:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
else:
print("Failed IP Lookup for: " + hostname)
original_record = dns_manager.find_one({"fqdn": hostname},
"mx")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
else:
print("Failed match on zone for: " + hostname)
# Record all the results.
dns_manager.remove_by_source("mx")
print("List length: " + str(len(input_list)))
for final_result in input_list:
dns_manager.insert_record(final_result, "mx")
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
# Obtain the list of known email addresses from the config collection
mongo_connector = MongoConnector.MongoConnector()
whois_collection = mongo_connector.get_whois_connection()
all_dns_collection = mongo_connector.get_all_dns_connection()
zones_collection = mongo_connector.get_zone_connection()
jobs_collection = mongo_connector.get_jobs_connection()
# Grab all zones that are not expired of false_positives
# Also exclude any that were recently created since they won't have data yet
date_delta = datetime.today() - timedelta(days=30)
zones = zones_collection.distinct('zone', {'created': {"$lt": date_delta}, 'status': {"$nin": [ZoneManager.EXPIRED, ZoneManager.FALSE_POSITIVE]}})
# The Python Whois library is hit and miss with some international zones.
# For now, this script focuses on the most popular TLDs.
new_zones = get_primary_zones(zones)
expired_list = []
for zone in new_zones:
if whois_collection.find({'zone': zone}).count() == 0:
# Assume it is expired if there is no longer a whois record present
expired_list.append(zone)
for zone in expired_list:
if all_dns_collection.find({'zone': zone}).count() > 0:
# This may be a case where the Python Whois library failed
# and the zone is still active.
print("DNS records still exist for " + zone)
expired_list.remove(zone)
zone_manager = ZoneManager(mongo_connector)
# Need to get this list before setting zones to expired in order to avoid a recursion problem.
already_expired = zone_manager.get_zones_by_status(ZoneManager.EXPIRED)
possibly_renewed = []
for zone in already_expired:
if whois_collection.find({'zone': zone}).count() == 1:
possibly_renewed.append(zone)
for zone in expired_list:
print("Expiring: " + zone)
zone_manager.set_status(zone, ZoneManager.EXPIRED, "mark_expired.py")
# Get the list of known registering entities.
# This will only work for some whois lookups since Python Whois doesn't get
# a valid org for all lookups and some have privacy enabled.
config_collection = mongo_connector.get_config_connection()
result = config_collection.find({}, {'Whois_Orgs': 1})
orgs = result[0]['Whois_Orgs']
for zone in possibly_renewed:
# We need to be careful of automatically marking something renewed
# since it could have been registered by someone else.
if whois_collection.find({'zone': zone, 'org': {"$in": orgs}}).count() == 1:
print("ATTENTION: " + zone + "has been renewed")
zone_manager.set_status(zone, ZoneManager.UNCONFIRMED, "mark_expired.py")
else:
print("WARNING: " + zone + " has been renewed by an unknown entity")
# Record status
jobs_collection.update_one({'job_name': 'mark_expired'},
{'$currentDate': {"updated": True},
"$set": {'status': 'COMPLETE'}})
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin Main()
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
jobs_collection = mongo_connector.get_jobs_connection()
mongo_ct = mongo_connector.get_certificate_transparency_connection()
cert_graphs_collection = mongo_connector.get_cert_graphs_connection()
zones = ZoneManager.get_distinct_zones(mongo_connector)
parser = argparse.ArgumentParser(description='Creates and stores certificate graphs in the database based on one or more sources.')
parser.add_argument('--check_censys', action='store_true', default=False, required=False, help='Whether to check the Censys collection in the database')
parser.add_argument('--check_443_scans', action='store_true', default=False, required=False, help='Whether to check the zgrab collection in the database')
parser.add_argument('--check_ct_scans', action='store_true', default=False, required=False, help='Whether to check the CT collection in the database')
args = parser.parse_args()
if args.check_censys is True:
censys_collection = mongo_connector.get_censys_connection()
if args.check_443_scans is True:
zgrab_collection = mongo_connector.get_zgrab_443_data_connection()
for zone in zones:
print("Creating: " + zone)
graph = nx.DiGraph()
certs_list = {}
if args.check_ct_scans:
certs_list = get_current_ct_certificates(mongo_ct, zone)
if args.check_censys:
certs_list = add_censys_certificates(censys_collection, zone, certs_list)
if args.check_443_scans:
certs_list = add_terminal_zgrab_certificates(zgrab_collection, zone, certs_list)
certs_list = add_initial_zgrab_certificates(zgrab_collection, zone, certs_list)
graph = create_nodes(graph, mongo_connector, zone, certs_list)
data = json_graph.node_link_data(graph)
my_data = {}
my_data['links'] = data['links']
my_data['nodes'] = data['nodes']
my_data['zone'] = zone
my_data['created'] = datetime.now()
cert_graphs_collection.remove({'zone': zone})
cert_graphs_collection.insert(my_data)
# Record status
jobs_collection.update_one({'job_name': 'create_cert_graphs'},
{'$currentDate': {"updated": True},
"$set": {'status': 'COMPLETE'}})
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin Main...
"""
global global_exit_flag
global global_zgrab_path
logger = LoggingUtil.create_log(__name__)
parser = argparse.ArgumentParser(
description='Launch zgrab against domains using port 80 or 443.')
parser.add_argument('-p',
choices=['443', '80'],
metavar="port",
help='The web port: 80 or 443')
parser.add_argument('-t',
default=5,
type=int,
metavar="threadCount",
help='The number of threads')
parser.add_argument('--zgrab_path',
default=global_zgrab_path,
metavar='zgrabVersion',
help='The version of ZGrab to use')
args = parser.parse_args()
if args.p == None:
logger.error("A port value (80 or 443) must be provided.")
exit(1)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
now = datetime.now()
logger.warning(str(now) + ": I am already running! Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
rm_connector = RemoteMongoConnector.RemoteMongoConnector()
all_dns_collection = rm_connector.get_all_dns_connection()
jobs_manager = JobsManager.JobsManager(rm_connector,
"zgrab_http_domain-" + args.p)
jobs_manager.record_job_start()
if args.p == "443":
zgrab_collection = rm_connector.get_zgrab_443_data_connection()
run_command = run_port_443_command
else:
zgrab_collection = rm_connector.get_zgrab_80_data_connection()
run_command = run_port_80_command
check_save_location("./json_p" + args.p)
global_zgrab_path = args.zgrab_path
zones = ZoneManager.get_distinct_zones(rm_connector)
ip_manager = IPManager.IPManager(rm_connector)
for zone in zones:
global_exit_flag = 0
domains = get_domains(all_dns_collection, ip_manager, zone)
if len(domains) == 0:
continue
num_threads = args.t
if len(domains) < args.t:
num_threads = len(domains)
logger.debug("Creating " + str(num_threads) + " threads")
threads = []
for thread_id in range(1, num_threads + 1):
thread = ZgrabThread(thread_id, global_work_queue, args.p,
run_command, zone, zgrab_collection)
thread.start()
threads.append(thread)
thread_id += 1
logger.debug(zone + " length: " + str(len(domains)))
logger.info("Populating Queue")
global_queue_lock.acquire()
for domain in domains:
global_work_queue.put(domain)
global_queue_lock.release()
# Wait for queue to empty
while not global_work_queue.empty():
pass
logger.info("Queue empty")
# Notify threads it's time to exit
global_exit_flag = 1
# Wait for all threads to complete
for t in threads:
t.join()
# Remove last week's old entries
lastweek = datetime.now() - timedelta(days=7)
zgrab_collection.remove({
'domain': {
"$ne": "<nil>"
},
'timestamp': {
"$lt": lastweek
}
})
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main()
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
mongo_ct = mongo_connector.get_certificate_transparency_connection()
cert_graphs_collection = mongo_connector.get_cert_graphs_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector,
"create_cert_graphs")
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
parser = argparse.ArgumentParser(
description=
"Creates and stores certificate graphs in the database based on one or more sources."
)
parser.add_argument(
"--check_censys",
action="store_true",
default=False,
required=False,
help="Whether to check the Censys collection in the database",
)
parser.add_argument(
"--check_443_scans",
action="store_true",
default=False,
required=False,
help="Whether to check the zgrab collection in the database",
)
parser.add_argument(
"--check_ct_scans",
action="store_true",
default=False,
required=False,
help="Whether to check the CT collection in the database",
)
parser.add_argument(
"--zgrab_version",
default=2,
type=int,
choices=[1, 2],
metavar="version",
help="The version of ZGrab used to collect data",
)
args = parser.parse_args()
if args.check_censys is True:
censys_collection = mongo_connector.get_censys_connection()
if args.check_443_scans is True:
zgrab_collection = mongo_connector.get_zgrab_443_data_connection()
for zone in zones:
logger.info("Creating: " + zone)
graph = nx.DiGraph()
certs_list = {}
if args.check_ct_scans:
certs_list = get_current_ct_certificates(mongo_ct, zone)
if args.check_censys:
certs_list = add_censys_certificates(censys_collection, zone,
certs_list)
if args.check_443_scans:
if args.zgrab_version == 1:
certs_list = add_terminal_zgrab_certificates(
zgrab_collection, zone, certs_list)
certs_list = add_initial_zgrab_certificates(
zgrab_collection, zone, certs_list)
else:
certs_list = add_terminal_zgrab2_certificates(
zgrab_collection, zone, certs_list)
certs_list = add_initial_zgrab2_certificates(
zgrab_collection, zone, certs_list)
graph = create_nodes(graph, mongo_connector, zone, certs_list)
data = json_graph.node_link_data(graph)
my_data = {}
my_data["links"] = data["links"]
my_data["nodes"] = data["nodes"]
my_data["zone"] = zone
my_data["created"] = datetime.now()
cert_graphs_collection.delete_one({"zone": zone})
mongo_connector.perform_insert(cert_graphs_collection, my_data)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Beging Main...
"""
global global_exit_flag
global global_retest_list
global global_sleep_time
global global_queue_size
global global_zgrab_path
logger = LoggingUtil.create_log(__name__)
global_retest_list = []
parser = argparse.ArgumentParser(
description="Launch zgrab against IPs using port 22, 25, 443, or 465.")
parser.add_argument(
"-p",
choices=["22", "25", "443", "465"],
metavar="port",
help="The port to scan: 22, 25, 443, or 465",
)
parser.add_argument("-t",
default=5,
type=int,
metavar="threadCount",
help="The number of threads")
parser.add_argument(
"--mx",
action="store_true",
help="Scan only IPs from MX records. Useful for SMTP scans.",
)
parser.add_argument(
"-s",
default=0,
type=int,
metavar="sleepTime",
help="Sleep time in order to spread out the batches",
)
parser.add_argument(
"--qs",
default=0,
type=int,
metavar="queueSize",
help="How many hosts to scan in a batch",
)
parser.add_argument("--zones_only",
action="store_true",
help="Scan only IPs from IP zones.")
parser.add_argument(
"--zgrab_path",
default=global_zgrab_path,
metavar="zgrabVersion",
help="The version of ZGrab to use",
)
args = parser.parse_args()
if args.p == None:
logger.error("A port value (22, 25, 443, or 465) must be provided.")
exit(1)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
now = datetime.now()
logger.warning(str(now) + ": I am already running! Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
rm_connector = RemoteMongoConnector.RemoteMongoConnector()
all_dns_collection = rm_connector.get_all_dns_connection()
ip_manager = IPManager.IPManager(rm_connector, True)
jobs_manager = JobsManager.JobsManager(rm_connector,
"zgrab_port_ip-" + args.p)
jobs_manager.record_job_start()
zones_struct = {}
zones_struct["zones"] = ZoneManager.get_distinct_zones(rm_connector)
# Not pretty but works
zones_struct["ip_manager"] = ip_manager
if args.mx:
(ips, ip_context) = get_mx_ips(zones_struct["zones"], ip_manager,
all_dns_collection)
elif args.zones_only:
(ips, ip_context) = get_only_ipzones(ip_manager.Tracked_CIDRs)
else:
(ips, ip_context) = get_ips(ip_manager, all_dns_collection)
if args.s and int(args.s) > 0:
global_sleep_time = int(args.s)
if args.qs and int(args.qs) > 0:
global_queue_size = int(args.qs)
logger.info("Got IPs: " + str(len(ips)))
zones_struct["ip_context"] = ip_context
zgrab_collection = rm_connector.get_zgrab_port_data_connection()
if args.p == "443":
run_command = run_port_443_command
elif args.p == "22":
run_command = run_port_22_command
elif args.p == "25":
run_command = run_port_25_command
elif args.p == "465":
run_command = run_port_465_command
check_save_location("./json_p" + args.p)
global_zgrab_path = args.zgrab_path
threads = []
logger.debug("Creating " + str(args.t) + " threads")
for thread_id in range(1, args.t + 1):
thread = ZgrabThread(
thread_id,
global_work_queue,
args.p,
run_command,
zones_struct,
zgrab_collection,
)
thread.start()
threads.append(thread)
thread_id += 1
logger.info("Populating Queue")
global_queue_lock.acquire()
for ip in ips:
global_work_queue.put(ip)
global_queue_lock.release()
# Wait for queue to empty
while not global_work_queue.empty():
pass
# Notify threads it's time to exit
global_exit_flag = 1
# Wait for all threads to complete
for t in threads:
t.join()
logger.info("Exiting Main Thread")
logger.info("Global retest list: " + str(len(global_retest_list)))
# Retest any SMTP hosts that did not respond to the StartTLS handshake
if args.p == "25" and len(global_retest_list) > 0:
process_thread(
logger,
global_retest_list,
args.p,
run_port_25_no_tls_command,
zones_struct,
zgrab_collection,
"retest",
)
# Remove old entries from before the scan
if args.p == "443":
other_results = zgrab_collection.find({
"data.tls": {
"$exists": True
},
"data.tls.timestamp": {
"$lt": now
}
})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result["_id"])},
{"$unset": {
"data.tls": ""
}})
elif args.p == "22":
if "zgrab2" in global_zgrab_path:
other_results = zgrab_collection.find({
"data.ssh": {
"$exists": True
},
"data.ssh.timestamp": {
"$lt": now
}
})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result["_id"])},
{"$unset": {
"data.ssh": ""
}})
else:
other_results = zgrab_collection.find({
"data.xssh": {
"$exists": True
},
"data.xssh.timestamp": {
"$lt": now
}
})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result["_id"])},
{"$unset": {
"data.xssh": ""
}})
elif args.p == "25":
other_results = zgrab_collection.find({
"data.smtp": {
"$exists": True
},
"data.smtp.timestamp": {
"$lt": now
}
})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result["_id"])},
{"$unset": {
"data.smtp": ""
}})
elif args.p == "465":
other_results = zgrab_collection.find({
"data.smtps": {
"$exists": True
},
"data.smtps.timestamp": {
"$lt": now
}
})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result["_id"])},
{"$unset": {
"data.smtps": ""
}})
# Remove any completely empty entries
zgrab_collection.delete_many({"data": {}})
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin main...
"""
parser = argparse.ArgumentParser(
description='Search the Common Crawl graph dataset for new domains')
parser.add_argument('--url',
metavar="URL",
help='The URL for the latest vertices file')
args = parser.parse_args()
if args.url != None:
CURRENT_FILE_LIST = args.url
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_collection = mongo_connector.get_jobs_connection()
reversed_zones = ZoneManager.get_reversed_zones(mongo_connector)
alphabet = list(string.digits + string.ascii_lowercase)
# Create a dictionary of the zones grouped by their first letter
# This will allow us to reduce the number of comparisons in the alphabetized CC files.
grouped_zones = {}
for letter in alphabet:
grouped_zones[letter] = []
for zone in reversed_zones:
first_letter = zone[0]
grouped_zones[first_letter].append(zone)
compressed_download_list = download_file(CURRENT_FILE_LIST)
subprocess.check_call(["gunzip", "-f", compressed_download_list])
download_list = compressed_download_list.split(".")[:-1]
list_file = ".".join(download_list)
vertices_file_entries = open(list_file, "r")
for entry in vertices_file_entries:
# Download file
vert_file_url = "http://commoncrawl.s3.amazonaws.com/" + entry.rstrip(
"\n")
compressed_vertices_file = download_file(vert_file_url)
# Decompress file
subprocess.check_call(["gunzip", "-f", compressed_vertices_file])
vertices_list = compressed_vertices_file.split(".")[:-1]
vertices_file = ".".join(vertices_list)
# Get the first and last line of the file
(first_line, last_line) = get_first_and_last_line(vertices_file)
# Get the first and last domain
parts = first_line.split("\t")
first_domain = parts[1].rstrip("\n")
first_char = first_domain[0]
parts = last_line.split("\t")
last_domain = parts[1].rstrip("\n")
last_char = last_domain[0]
# Get the list of zones relevant to that range
searchable_zones = get_zone_sublist(first_char, last_char,
grouped_zones)
# Parse file and insert matches
parse_file(vertices_file, searchable_zones, dns_manager)
subprocess.check_call(["rm", vertices_file])
# Remove all entries more than two months old
# Note: This commented out because Common Crawl graph data is not additive.
# dns_manager.remove_all_by_source_and_date("common_crawl", -4)
jobs_collection.update_one({'job_name': 'common_crawl_graph'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Ending: " + str(now))
def main():
now = datetime.now()
print("Starting: " + str(now))
azure_connector = AzureConnector.AzureConnector()
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zone_ingestor = ZoneIngestor.ZoneIngestor()
jobs_manager = JobsManager.JobsManager(mongo_connector, 'fetch_azure_dns')
jobs_manager.record_job_start()
current_zones = ZoneManager.get_distinct_zones(mongo_connector)
resource_client = azure_connector.get_resources_client()
resources = []
# The resource list is not currently used.
for item in resource_client.resource_groups.list():
resources.append(item.name)
dns_client = azure_connector.get_dns_client()
zones = dns_client.zones.list()
# The type of records the Azure DNS will let you configure
record_types = {
'A': 'arecords',
'AAAA': 'aaaa_records',
'MX': 'mx_records',
'NS': 'ns_records',
'PTR': 'ptr_records',
'SRV': 'srv_records',
'TXT': 'txt_records',
'CNAME': 'cname_record',
'SOA': 'soa_record'
}
for zone in zones:
print("Zone: " + zone.name)
data = split_id(zone.id)
if zone.zone_type == ZoneType.public:
print(zone.name + " is public:")
if zone.name not in current_zones:
print("Creating zone: " + zone.name)
zone_ingestor.add_zone(zone.name,
"azure:" + data["resourceGroups"])
try:
print("ResourceGroup: " + data["resourceGroups"])
records = dns_client.record_sets.list_all_by_dns_zone(
data["resourceGroups"], zone.name)
for entry in records:
# The record_data id value ends in rtype/rvalue so you must guess the rtype
record_data = split_id(entry.id)
for rtype in record_types:
if rtype in record_data:
results = extract_record_set_value(rtype, entry)
for result in results:
result['zone'] = zone.name
result['created'] = datetime.now()
result['status'] = 'confirmed'
dns_manager.insert_record(
result, "azure:" + data["resourceGroups"])
except:
print("No records found")
jobs_manager.record_job_complete()
def main():
"""
Begin Main...
"""
global global_exit_flag
global global_zgrab_path
logger = LoggingUtil.create_log(__name__)
parser = argparse.ArgumentParser(
description="Launch zgrab against IPs using port 80 or 443.")
parser.add_argument("-p",
choices=["443", "80"],
metavar="port",
help="The web port: 80 or 443")
parser.add_argument("-t",
default=5,
type=int,
metavar="threadCount",
help="The number of threads")
parser.add_argument(
"--zgrab_path",
default=global_zgrab_path,
metavar="zgrabVersion",
help="The version of ZGrab to use",
)
args = parser.parse_args()
if args.p == None:
logger.error("A port value (80 or 443) must be provided.")
exit(1)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
now = datetime.now()
logger.warning(str(now) + ": I am already running! Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
rm_connector = RemoteMongoConnector.RemoteMongoConnector()
all_dns_collection = rm_connector.get_all_dns_connection()
ip_manager = IPManager.IPManager(rm_connector, True)
jobs_manager = JobsManager.JobsManager(rm_connector,
"zgrab_http_ip-" + args.p)
jobs_manager.record_job_start()
zones_struct = {}
zones_struct["zones"] = ZoneManager.get_distinct_zones(rm_connector)
# Not pretty but cleaner than previous method
zones_struct["ip_manager"] = ip_manager
(ips, ip_context) = get_ips(ip_manager, all_dns_collection)
logger.info("Got IPs: " + str(len(ips)))
zones_struct["ip_context"] = ip_context
if args.p == "443":
zgrab_collection = rm_connector.get_zgrab_443_data_connection()
run_command = run_port_443_command
else:
zgrab_collection = rm_connector.get_zgrab_80_data_connection()
run_command = run_port_80_command
check_save_location("./json_p" + args.p)
global_zgrab_path = args.zgrab_path
threads = []
logger.debug("Creating " + str(args.t) + " threads")
for thread_id in range(1, args.t + 1):
thread = ZgrabThread(
thread_id,
global_work_queue,
args.p,
run_command,
zones_struct,
zgrab_collection,
)
thread.start()
threads.append(thread)
thread_id += 1
logger.info("Populating Queue")
global_queue_lock.acquire()
for ip in ips:
global_work_queue.put(ip)
global_queue_lock.release()
# Wait for queue to empty
while not global_work_queue.empty():
pass
# Notify threads it's time to exit
global_exit_flag = 1
# Wait for all threads to complete
for t in threads:
t.join()
logger.info("Exiting Main Thread")
# Remove last week's old entries
lastweek = datetime.now() - timedelta(days=7)
zgrab_collection.delete_many({
"ip": {
"$ne": "<nil>"
},
"timestamp": {
"$lt": lastweek
}
})
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
if is_running(os.path.basename(__file__)):
logger.warning("Already running...")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
parser = argparse.ArgumentParser(
description="Parse Sonar files based on domain zones.")
parser.add_argument(
"--sonar_file_type",
choices=["dns-any", "dns-a", "rdns"],
required=True,
help='Specify "dns-any", "dns-a", or "rdns"',
)
parser.add_argument(
"--database",
choices=["local", "remote"],
required=False,
default="local",
help="Whether to use the local or remote DB",
)
args = parser.parse_args()
r7 = Rapid7.Rapid7()
if args.database == "remote":
mongo_connector = RemoteMongoConnector.RemoteMongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector,
"get_sonar_data_dns")
else:
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zones = ZoneManager.get_distinct_zones(mongo_connector)
save_directory = "./files/"
check_save_location(save_directory)
# A session is necessary for the multi-step log-in process
s = requests.Session()
if args.sonar_file_type == "rdns":
logger.info("Updating RDNS records")
jobs_manager = JobsManager.JobsManager(mongo_connector,
"get_sonar_data_rdns")
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "rdns", jobs_manager)
if html_parser.rdns_url == "":
logger.error("Unknown Error")
jobs_manager.record_job_error()
exit(0)
unzipped_rdns = download_remote_files(logger, s,
html_parser.rdns_url,
save_directory, jobs_manager)
update_rdns(logger, unzipped_rdns, zones, dns_manager,
mongo_connector)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns-any":
logger.info("Updating DNS ANY records")
jobs_manager = JobsManager.JobsManager(mongo_connector,
"get_sonar_data_dns-any")
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.any_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.any_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, dns_manager)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns-a":
logger.info("Updating DNS A, AAAA, and CNAME records")
jobs_manager = JobsManager.JobsManager(mongo_connector,
"get_sonar_data_dns-a")
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.a_url != "":
logger.info("Updating A records")
unzipped_dns = download_remote_files(logger, s,
html_parser.a_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, dns_manager)
if html_parser.aaaa_url != "":
logger.info("Updating AAAA records")
unzipped_dns = download_remote_files(logger, s,
html_parser.aaaa_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, dns_manager)
if html_parser.cname_url != "":
logger.info("Updating CNAME records")
unzipped_dns = download_remote_files(logger, s,
html_parser.cname_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, dns_manager)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
jobs_manager.record_job_complete()
else:
logger.error("Unrecognized sonar_file_type option. Exiting...")
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Set up the common objects
mongo_connector = MongoConnector.MongoConnector()
ct_collection = mongo_connector.get_certificate_transparency_connection()
zones = ZoneManager.get_distinct_zones(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, "get_crt_sh")
jobs_manager.record_job_start()
save_location = "/mnt/workspace/crt_sh"
download_method = 'dbAndSave'
parser = argparse.ArgumentParser(
description='Download DNS and/or certificate information from crt.sh.')
parser.add_argument(
'--fetch_dns_records',
action='store_true',
help='Indicates whether to add DNS entries to the database')
parser.add_argument(
'--download_methods',
choices=['dbAndSave', 'dbOnly'],
default=download_method,
help=
'Indicates whether to download the raw files or just record in the database.'
)
parser.add_argument(
'--cert_save_location',
required=False,
default=save_location,
help=
'Indicates where to save the certificates on disk when choosing dbAndSave'
)
args = parser.parse_args()
if args.cert_save_location:
save_location = args.cert_save_location
if not save_location.endswith("/"):
save_location = save_location + "/"
if args.download_methods == 'dbAndSave':
check_save_location(save_location)
for zone in zones:
# Pace out requests so as not to DoS crt.sh and Google DNS
time.sleep(5)
# This could be done with backoff but we don't want to be overly aggressive.
json_result = make_https_request(
logger, "https://crt.sh/?q=%25." + zone + "&output=json")
if json_result is None:
logger.warning("Can't find result for: " + zone)
json_result = "{}"
json_data = json.loads(json_result)
new_names = []
new_ids = []
for entry in json_data:
if entry['id'] not in new_ids:
new_ids.append(entry['id'])
if "*" not in entry["name_value"] and entry[
"name_value"] not in new_names:
new_names.append(entry["name_value"])
if args.fetch_dns_records:
add_new_domain_names(new_names, zones, mongo_connector)
if args.download_methods == "dbAndSave":
add_new_certificate_values(logger, new_ids, ct_collection, zones,
save_location)
elif args.download_methods == "dbOnly":
add_new_certificate_values(logger, new_ids, ct_collection, zones,
None)
# Set isExpired for any entries that have recently expired.
ct_collection.update(
{
"not_after": {
"$lt": datetime.utcnow()
},
"isExpired": False
}, {"$set": {
"isExpired": True
}},
multi=True)
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin main...
"""
if is_running("get_censys_files.py"):
"""
Check to see if a download is in process...
"""
now = datetime.now()
print(str(now) + ": Can't run due to get_files running. Goodbye!")
exit(0)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
now = datetime.now()
print(str(now) + ": I am already running! Goodbye!")
exit(0)
# Make the relevant database connections
RMC = RemoteMongoConnector.RemoteMongoConnector()
# Verify that the get_files script has a recent file in need of parsing.
jobs_collection = RMC.get_jobs_connection()
status = jobs_collection.find_one({'job_name': 'censys'})
if status['status'] != "DOWNLOADED":
now = datetime.now()
print(str(now) + ": The status is not set to DOWNLOADED. Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
# Collect the list of available zones
zones = ZoneManager.get_distinct_zones(RMC)
print("Zones: " + str(len(zones)))
# Collect the list of AWS CIDRs
aws_ips = []
get_aws_ips(RMC, aws_ips)
print("AWS IPs: " + str(len(aws_ips)))
# Collect the list of Azure CIDRs
azure_ips = []
get_azure_ips(RMC, azure_ips)
print("Azure IPs: " + str(len(azure_ips)))
# Collect the list of known CIDRs
ip_zones_collection = RMC.get_ipzone_connection()
results = ip_zones_collection.find({'status': {"$ne": "false_positive"}})
cidrs = []
for entry in results:
cidrs.append(IPNetwork(entry['zone']))
print("CIDRs: " + str(len(cidrs)))
# Get the current configuration information for Marinus.
config_collection = RMC.get_config_connection()
configs = config_collection.find({})
orgs = []
for org in configs[0]['SSL_Orgs']:
orgs.append(org)
print("Orgs: " + str(len(orgs)))
# Obtain the name of the decompressed file.
filename_f = open(FILENAME_FILE, "r")
decompressed_file = filename_f.readline()
filename_f.close()
# For manual testing: decompressed_file = "ipv4.json"
now = datetime.now()
print(str(now) + ": Beginning file processing...")
# Remove old results from the database
results_collection = RMC.get_results_connection()
results_collection.remove({})
all_dns_collection = RMC.get_all_dns_connection()
try:
with open(decompressed_file, "r") as dec_f:
for line in dec_f:
try:
entry = json.loads(line)
"""
Does the SSL certificate match a known organization?
Is the IP address in a known CIDR?
"""
if check_in_org(entry, orgs) or \
check_in_cidr(entry['ip'], cidrs):
entry['zones'] = check_in_zone(entry, zones)
entry['aws'] = is_aws_ip(entry['ip'], aws_ips)
entry['azure'] = is_azure_ip(entry['ip'], azure_ips)
(domains,
zones) = lookup_domain(entry, zones,
all_dns_collection)
if len(domains) > 0:
entry['domains'] = domains
if len(zones) > 0:
for zone in zones:
if zone not in entry['zones']:
entry['zones'].append(zone)
insert_result(entry, results_collection)
# else:
# #This will add days to the amount of time necessary to scan the file.
# matched_zones = check_in_zone(entry, zones)
# if matched_zones != []:
# entry['zones'] = matched_zones
# entry['aws'] = is_aws_ip(entry['ip'], aws_ips)
# entry['azure'] = is_azure_ip(entry['ip'], azure_ips)
# insert_result(entry, results_collection)
except ValueError as err:
print("Value Error!")
print(str(err))
except:
print("Line unexpected error:", sys.exc_info()[0])
print("Line unexpected error:", sys.exc_info()[1])
except IOError as err:
print("I/O error({0}): {1}".format(err.errno, err.strerror))
exit(0)
except:
print("Unexpected error:", sys.exc_info()[0])
print("Unexpected error:", sys.exc_info()[1])
exit(0)
# Indicate that the processing of the job is complete and ready for download to Marinus
jobs_collection.update_one({'job_name': 'censys'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin Main...
"""
# The sources for which to remove expired entries
# Infoblox is handled separately
# {"source_name": date_difference_in_months}
sources = [{
"name": "sonar_dns",
"diff": -2
}, {
"name": "sonar_dns_saved",
"diff": -2
}, {
"name": "sonar_rdns",
"diff": -2
}, {
"name": "sonar_rdns_saved",
"diff": -2
}, {
"name": "ssl",
"diff": -2
}, {
"name": "ssl_saved",
"diff": -2
}, {
"name": "virustotal",
"diff": -2
}, {
"name": "virustotal_saved",
"diff": -2
}, {
"name": "UltraDNS",
"diff": -2
}, {
"name": "UltraDNS_saved",
"diff": -2
}, {
"name": "marinus",
"diff": -2
}, {
"name": "marinus_saved",
"diff": -2
}, {
"name": "mx",
"diff": -2
}, {
"name": "mx_saved",
"diff": -2
}, {
"name": "common_crawl",
"diff": -4
}, {
"name": "common_crawl_saved",
"diff": -4
}]
amass_diff = -2
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
dns_manager = DNSManager.DNSManager(mongo_connector)
GDNS = GoogleDNS.GoogleDNS()
ip_manager = IPManager.IPManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
'remove_expired_entries')
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
# Get the date for today minus two months
d_minus_2m = monthdelta(datetime.now(), -2)
print("Removing SRDNS as of: " + str(d_minus_2m))
# Remove the old records
srdns_collection = mongo_connector.get_sonar_reverse_dns_connection()
srdns_collection.remove({'updated': {"$lt": d_minus_2m}})
ip_manager.delete_records_by_date(d_minus_2m)
# Before completely removing old entries, make an attempt to see if they are still valid.
# Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source.
# Rather than throw away valid information, this will archive it.
for entry in sources:
removal_date = monthdelta(datetime.now(), entry['diff'])
source = entry['name']
print("Removing " + source + " as of: " + str(removal_date))
last_domain = ""
results = all_dns_collection.find({
'sources': {
"$size": 1
},
'sources.source': source,
'sources.updated': {
"$lt": removal_date
}
})
for result in results:
if result['fqdn'] != last_domain:
last_domain = result['fqdn']
lookup_int = get_lookup_int(result, GDNS)
dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int)
if dns_result != []:
insert_current_results(dns_result, dns_manager, zones,
result, source)
dns_manager.remove_all_by_source_and_date(source, entry['diff'])
# Process amass entries
temp_sources = mongo_connector.perform_distinct(all_dns_collection,
'sources.source')
amass_sources = []
for entry in temp_sources:
if entry.startswith("amass:"):
amass_sources.append(entry)
for source in amass_sources:
removal_date = monthdelta(datetime.now(), amass_diff)
print("Removing " + source + " as of: " + str(removal_date))
last_domain = ""
results = mongo_connector.perform_find(
all_dns_collection, {
'sources': {
"$size": 1
},
'sources.source': source,
'sources.updated': {
"$lt": removal_date
}
})
for result in results:
if result['fqdn'] != last_domain:
last_domain = result['fqdn']
lookup_int = get_lookup_int(result, GDNS)
dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int)
if dns_result != []:
insert_current_results(dns_result, dns_manager, zones,
result, source)
dns_manager.remove_all_by_source_and_date(source, amass_diff)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
Begin main...
"""
logger = LoggingUtil.create_log(__name__)
if is_running("get_censys_files.py"):
"""
Check to see if a download is in process...
"""
logger.warning("Can't run due to get_files running. Goodbye!")
exit(0)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
logger.warning("I am already running! Goodbye!")
exit(0)
# Make the relevant database connections
RMC = RemoteMongoConnector.RemoteMongoConnector()
ip_manager = IPManager.IPManager(RMC)
# Verify that the get_files script has a recent file in need of parsing.
jobs_collection = RMC.get_jobs_connection()
status = jobs_collection.find_one({"job_name": "censys"})
if status["status"] != "DOWNLOADED":
logger.warning("The status is not set to DOWNLOADED. Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Collect the list of available zones
zones = ZoneManager.get_distinct_zones(RMC)
logger.info("Zones: " + str(len(zones)))
# Get the current configuration information for Marinus.
config_collection = RMC.get_config_connection()
configs = config_collection.find({})
orgs = []
for org in configs[0]["SSL_Orgs"]:
orgs.append(org)
logger.info("Orgs: " + str(len(orgs)))
# Obtain the name of the decompressed file.
filename_f = open(FILENAME_FILE, "r")
decompressed_file = filename_f.readline()
filename_f.close()
# For manual testing: decompressed_file = "ipv4.json"
logger.info("Beginning file processing...")
# Remove old results from the database
results_collection = RMC.get_results_connection()
results_collection.delete_many({})
all_dns_collection = RMC.get_all_dns_connection()
try:
with open(decompressed_file, "r") as dec_f:
for line in dec_f:
try:
entry = json.loads(line)
"""
Does the SSL certificate match a known organization?
Is the IP address in a known CIDR?
Is the IP address recorded in Splunk?
"""
if (check_in_org(entry, orgs)
or ip_manager.is_tracked_ip(entry["ip"])
or ip_manager.find_splunk_data(entry["ip"], "AWS")
is not None or ip_manager.find_splunk_data(
entry["ip"], "AZURE") is not None):
entry["zones"] = check_in_zone(entry, zones)
entry["aws"] = ip_manager.is_aws_ip(entry["ip"])
entry["azure"] = ip_manager.is_azure_ip(entry["ip"])
(domains,
zones) = lookup_domain(entry, zones,
all_dns_collection)
if len(domains) > 0:
entry["domains"] = domains
if len(zones) > 0:
for zone in zones:
if zone not in entry["zones"]:
entry["zones"].append(zone)
insert_result(entry, results_collection)
# else:
# #This will add days to the amount of time necessary to scan the file.
# matched_zones = check_in_zone(entry, zones)
# if matched_zones != []:
# entry['zones'] = matched_zones
# entry['aws'] = ip_manager.is_aws_ip(entry['ip'])
# entry['azure'] = ip_manager.is_azure_ip(entry['ip'])
# insert_result(entry, results_collection)
except ValueError as err:
logger.error("Value Error!")
logger.error(str(err))
except:
logger.error("Line unexpected error: " +
str(sys.exc_info()[0]))
logger.error("Line unexpected error: " +
str(sys.exc_info()[1]))
except IOError as err:
logger.error("I/O error({0}): {1}".format(err.errno, err.strerror))
exit(1)
except:
logger.error("Unexpected error: " + str(sys.exc_info()[0]))
logger.error("Unexpected error: " + str(sys.exc_info()[1]))
exit(1)
# Indicate that the processing of the job is complete and ready for download to Marinus
jobs_collection.update_one(
{"job_name": "censys"},
{
"$currentDate": {
"updated": True
},
"$set": {
"status": "COMPLETE"
}
},
)
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
