def main():
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
splunk_query_manager = SplunkQueryManager.SplunkQueryManager()
splunk_collection = mongo_connector.get_splunk_connection()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, "get_splunk_data")
jobs_manager.record_job_start()
print("Starting Splunk Query")
results_per_page = 100
# Put your custom Splunk search query here.
results = splunk_query_manager.do_search('search index=...',
results_per_page)
print("Processing " + str(splunk_query_manager.RESULTCOUNT) + " results")
parse_splunk_results(results, dns_manager, splunk_collection)
while True:
results = splunk_query_manager.get_next_page()
if results is None:
break
parse_splunk_results(results, dns_manager, splunk_collection)
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
def create_nodes(graph, mongo_connector, zone, all_certs):
"""
Create the list of D3.js nodes and links based on the collected certificates
"""
DNS_MGR = DNSManager.DNSManager(mongo_connector)
for cert in all_certs:
matched_count = 0
if "censys_count" in cert:
matched_count = cert['censys_count']
if "zgrab_count" in cert:
matched_count = matched_count + cert['zgrab_count']
graph.add_node(cert['id'], type='certificate', sources=cert['sources'], total_count=matched_count)
for dns_entry in cert['dns_entries']:
lookup = DNS_MGR.find_one({'fqdn': dns_entry}, None)
root_flag = "false"
if dns_entry == zone:
root_flag = "true"
if lookup is None:
graph.add_node(dns_entry, root=root_flag, status="No Host", type="domain", sources=cert['sources'])
else:
graph.add_node(dns_entry, root=root_flag, status="Resolves", type="domain", sources=cert['sources'])
graph.add_edge(cert['id'], dns_entry, type="sans")
graph.add_edge(dns_entry, cert['id'], type="uses")
return graph
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print ("Starting: " + str(now))
logger.info("Starting...")
dns_types = {"a":1, "ns":2, "cname":5, "soa":6, "ptr":12, "hinfo": 13, "mx": 15, "txt":16, "aaaa":28, "srv":33, "naptr": 35, "ds": 43, "rrsig": 46, "dnskey": 48}
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector, 'marinus_dns')
jobs_manager.record_job_start()
dns_manager = DNSManager.DNSManager(mongo_connector)
zones = ZoneManager.get_distinct_zones(mongo_connector)
google_dns = GoogleDNS.GoogleDNS()
for zone in zones:
time.sleep(1)
for dtype, dnum in dns_types.items():
result = google_dns.fetch_DNS_records(zone, dnum)
if result == []:
logger.debug("No records found for " + zone)
else:
new_record = result[0]
new_record['status'] = 'confirmed'
new_record['zone'] = zone
new_record['created'] = datetime.now()
logger.debug ("Found " + dtype + " for: " + zone)
dns_manager.insert_record(new_record, "marinus")
logger.info("Starting SOA Search")
soa_searches = find_sub_zones(all_dns_collection)
for entry in soa_searches:
time.sleep(1)
result = google_dns.fetch_DNS_records(zone, dns_types['soa'])
if result != []:
new_record = result[0]
new_record['status'] = 'confirmed'
new_record['zone'] = get_fld_from_value(entry, '')
new_record['created'] = datetime.now()
logger.debug ("Found SOA: " + entry)
if new_record['zone'] != '':
dns_manager.insert_record(new_record, "marinus")
jobs_manager.record_job_complete()
now = datetime.now()
print ("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_collection = mongo_connector.get_jobs_connection()
zone_ingestor = ZoneIngestor.ZoneIngestor()
current_zones = ZoneManager.get_distinct_zones(mongo_connector)
# For cases with multiple R53 accounts, include the account id for reference
sts = boto3.client('sts')
account_id = sts.get_caller_identity()["Arn"].split(':')[4]
r53_source = "R53:" + str(account_id)
r53_client = boto3.client('route53')
r53_domains = r53_client.list_hosted_zones()
r53_zone_list = []
while r53_domains != {}:
for zone_data in r53_domains['HostedZones']:
# Only add public zones
if zone_data['Config']['PrivateZone'] == False:
r53_zone_list.append(zone_data)
if r53_domains['IsTruncated'] == True:
r53_domains = r53_client.list_domains(
Marker=r53_domains['NextMarker'])
else:
r53_domains = {}
for zone_data in r53_zone_list:
# Double check that this is not a new zone
zone_name = zone_data['Name'][:-1]
if zone_name not in current_zones:
print("Creating zone: " + zone_name)
zone_ingestor.add_zone(zone_data['Name'], r53_source)
# Add hosts to the zone
update_records(r53_client, dns_manager, zone_data, r53_source)
# Record status
jobs_collection.update_one({'job_name': 'get_route53'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zone_ingestor = ZoneIngestor.ZoneIngestor()
jobs_manager = JobsManager.JobsManager(mongo_connector, "get_route53")
jobs_manager.record_job_start()
current_zones = ZoneManager.get_distinct_zones(mongo_connector)
# For cases with multiple R53 accounts, include the account id for reference
sts = boto3.client("sts")
account_id = sts.get_caller_identity()["Arn"].split(":")[4]
r53_source = "R53:" + str(account_id)
r53_client = boto3.client("route53")
r53_domains = r53_client.list_hosted_zones()
r53_zone_list = []
while r53_domains != {}:
for zone_data in r53_domains["HostedZones"]:
# Only add public zones
if zone_data["Config"]["PrivateZone"] == False:
r53_zone_list.append(zone_data)
if r53_domains["IsTruncated"] == True:
r53_domains = r53_client.list_domains(
Marker=r53_domains["NextMarker"])
else:
r53_domains = {}
for zone_data in r53_zone_list:
# Double check that this is not a new zone
zone_name = zone_data["Name"][:-1]
if zone_name not in current_zones:
logger.info("Creating zone: " + zone_name)
zone_ingestor.add_zone(zone_data["Name"], r53_source)
# Add hosts to the zone
update_records(r53_client, dns_manager, zone_data, r53_source)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
"get_external_cnames")
jobs_manager.record_job_start()
groups = {}
# Collect zones
zone_results = ZoneManager.get_distinct_zones(mongo_connector)
zones = []
for zone in zone_results:
if zone.find(".") >= 0:
zones.append(zone)
# Collect the all_dns cnames.
logger.info("Starting All DNS...")
all_dns_recs = dns_manager.find_multiple({"type": "cname"}, None)
for srec in all_dns_recs:
if not is_tracked_zone(srec["value"], zones):
add_to_list(
get_fld_from_value(srec["value"], srec["zone"]),
srec["fqdn"],
srec["value"],
srec["zone"],
groups,
)
# Update the database
tpds_collection = mongo_connector.get_tpds_connection()
tpds_collection.delete_many({})
for key in groups.keys():
tpds_collection.insert_one(groups[key])
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
now = datetime.now()
print ("Starting: " + str(now))
dns_types = {"a":1, "ns":2, "cname":5, "soa":6, "ptr":12, "hinfo": 13, "mx": 15, "txt":16, "aaaa":28, "srv":33, "naptr": 35, "ds": 43, "rrsig": 46, "dnskey": 48}
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
jobs_collection = mongo_connector.get_jobs_connection()
dns_manager = DNSManager.DNSManager(mongo_connector)
zones = ZoneManager.get_distinct_zones(mongo_connector)
google_dns = GoogleDNS.GoogleDNS()
for zone in zones:
time.sleep(1)
for dtype, dnum in dns_types.items():
result = google_dns.fetch_DNS_records(zone, dnum)
if result == []:
print("No records found for " + zone)
else:
new_record = result[0]
new_record['status'] = 'confirmed'
new_record['zone'] = zone
new_record['created'] = datetime.now()
print ("Found " + dtype + " for: " + zone)
dns_manager.insert_record(new_record, "marinus")
print("Starting SOA Search")
soa_searches = find_sub_zones(all_dns_collection)
for entry in soa_searches:
time.sleep(1)
result = google_dns.fetch_DNS_records(zone, dns_types['soa'])
if result != []:
new_record = result[0]
new_record['status'] = 'confirmed'
new_record['zone'] = get_fld_from_value(entry, '')
new_record['created'] = datetime.now()
print ("Found SOA: " + entry)
if new_record['zone'] != '':
dns_manager.insert_record(new_record, "marinus")
jobs_collection.update_one({'job_name': 'marinus_dns'},
{'$currentDate': {"updated" : True},
"$set": {'status': 'COMPLETE'}})
now = datetime.now()
print ("Complete: " + str(now))
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_collection = mongo_connector.get_jobs_connection()
groups = {}
# Collect zones
zone_results = ZoneManager.get_distinct_zones(mongo_connector)
zones = []
for zone in zone_results:
if zone.find(".") >= 0:
zones.append(zone)
# Collect the all_dns cnames.
print("Starting All DNS...")
all_dns_recs = dns_manager.find_multiple({'type': 'cname'}, None)
for srec in all_dns_recs:
if not is_tracked_zone(srec['value'], zones):
add_to_list(get_fld_from_value(srec['value'], srec['zone']),
srec['fqdn'], srec['value'], srec['zone'], groups)
# Update the database
tpds_collection = mongo_connector.get_tpds_connection()
tpds_collection.remove({})
for key in groups.keys():
tpds_collection.insert(groups[key])
# Record status
jobs_collection.update_one({'job_name': 'get_external_cnames'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Ending: " + str(now))
def find_dns_zones(self, ip):
"""
Find DNS zones related to the IP address.
"""
dns_manager = DNSManager.DNSManager(self.mongo_connector)
results = dns_manager.find_multiple({"value": ip}, None)
zones = []
domains = []
for result in results:
if result['zone'] not in zones and result['zone'] != '':
zones.append(result['zone'])
if result['fqdn'] not in domains and result['fqdn'] != ip:
domains.append(result['fqdn'])
return zones, domains
def add_new_domain_names(hostnames, zones, mongo_connector):
"""
Perform a GoogleDNS lookup on all identified domain names
and add them to the DNS tracker.
"""
google_dns = GoogleDNS.GoogleDNS()
dns_manager = DNSManager.DNSManager(mongo_connector)
for hostname in hostnames:
results = google_dns.fetch_DNS_records(hostname)
if results != []:
for result in results:
temp_zone = get_tracked_zone(result['fqdn'], zones)
if temp_zone is not None:
new_record = {"fqdn": result['fqdn']}
new_record['zone'] = temp_zone
new_record['created'] = datetime.now()
new_record['type'] = result['type']
new_record['value'] = result['value']
new_record['status'] = 'unknown'
dns_manager.insert_record(new_record, "ssl")
def main():
"""
Begin Main
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Set up all the database connections
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
'create_netaddr_graphs')
jobs_manager.record_job_start()
# Get the list of the all Class C's in Marinus
groups = []
create_list_of_cidrs(groups, mongo_connector, dns_manager)
# Create a separate copy of the class C list since groups will be modified later
cidr_list = groups + []
# Create the stats on the network data
group_data = create_network_data_sets(groups, mongo_connector)
logger.info("Number of Tracked Class C's: " +
str(group_data['tracked_count']))
logger.info("Number of AWS Class C's: " + str(group_data['aws_count']))
logger.info("Number of Azure Class C's: " + str(group_data['azure_count']))
logger.info("Number of Akamai Class C's: " +
str(group_data['akamai_count']))
logger.info("Number of Class C's: " + str(len(groups)))
# Get the current list of zones
zones = ZoneManager.get_distinct_zones(mongo_connector)
# For each Class C that was identified in Marinus...
for tcidr in cidr_list:
cidr = tcidr.replace(REPLACE_CHAR, ".")
groups = []
graph = nx.Graph()
add_to_list(cidr, groups)
graph.add_node(cidr,
data_type="class_c",
type=0,
depends=[],
dependedOnBy=[],
docs="<h1>Parent</h1>")
find_all_dns_by_zone(graph, cidr, groups, dns_manager)
find_srdns_by_zone(graph, cidr, groups, mongo_connector)
data = json_graph.node_link_data(graph)
reformat_data(data, cidr, groups)
new_data = {}
new_data['directed'] = data['directed']
new_data['graph'] = data['graph']
new_data['multigraph'] = data['multigraph']
new_data['errs'] = []
new_data['links'] = data['links']
new_data['data'] = {}
for i in range(0, len(data['nodes'])):
new_data['data'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)] = data['nodes'][i]
config = {}
config['title'] = cidr + " Network Map"
config['graph'] = {}
config['graph']['linkDistance'] = 150
config['graph']['charge'] = -400
config['graph']['height'] = 800
config['graph']['numColors'] = len(groups)
config['graph']['labelPadding'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['labelMargin'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['ticksWithoutCollisions'] = 50
config['graph_type'] = "cidr"
config['types'] = {}
regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
regx = re.compile(regex_str)
for tgroup in groups:
group = tgroup.replace(REPLACE_CHAR, ".")
data_type = "tpd"
if group in zones:
data_type = "tracked_domain"
elif re.match(regx, group):
data_type = "cidr"
config['types'][tgroup] = {
"short": group,
"long": "A group from the network: " + group,
"data_type": data_type
}
config['constraints'] = []
tmp = int(math.ceil(math.sqrt(len(groups)))) + 1
x = []
y = []
for i in range(1, tmp):
val = round((i * 1.0) / tmp, 2)
x.append(str(val))
y.append(str(val))
x_pos = 0
y_pos = 0
for group in groups:
config['constraints'].append({
"has": {
"type": group
},
"type": "position",
"x": x[x_pos],
"y": y[y_pos]
})
x_pos = x_pos + 1
if x_pos >= len(x):
x_pos = 0
y_pos = y_pos + 1
config['jsonUrl'] = "/api/v1.0/cidr_graphs/" + cidr
new_data['config'] = config
new_data['created'] = datetime.now()
new_data['zone'] = cidr
cidr_graphs_collection = mongo_connector.get_cidr_graphs_connection()
cidr_graphs_collection.remove({'zone': cidr})
cidr_graphs_collection.insert_one(new_data)
time.sleep(1)
# Remove last week's old entries
lastweek = datetime.now() - timedelta(days=7)
cidr_graphs_collection.remove({'created': {"$lt": lastweek}})
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
dns_manager = DNSManager.DNSManager(mongo_connector)
GDNS = GoogleDNS.GoogleDNS()
ip_manager = IPManager.IPManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
'remove_expired_entries')
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
# The sources for which to remove expired entries
results = mongo_connector.perform_distinct(all_dns_collection,
'sources.source')
sources = []
for source in results:
temp = {}
temp['name'] = source
if "common_crawl" in source:
temp['diff'] = -4
else:
temp['diff'] = -2
sources.append(temp)
# Before completely removing old entries, make an attempt to see if they are still valid.
# Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source.
# Rather than throw away valid information, this will archive it.
for entry in sources:
removal_date = monthdelta(datetime.now(), entry['diff'])
source = entry['name']
logger.debug("Removing " + source + " as of: " + str(removal_date))
last_domain = ""
results = all_dns_collection.find({
'sources': {
"$size": 1
},
'sources.source': source,
'sources.updated': {
"$lt": removal_date
}
})
for result in results:
if result['fqdn'] != last_domain:
last_domain = result['fqdn']
lookup_int = get_lookup_int(logger, result, GDNS)
dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int)
if dns_result != []:
insert_current_results(dns_result, dns_manager, zones,
result, source)
dns_manager.remove_all_by_source_and_date(source, entry['diff'])
# Get the date for today minus two months
d_minus_2m = monthdelta(datetime.now(), -2)
logger.info("Removing SRDNS as of: " + str(d_minus_2m))
# Remove the old records
srdns_collection = mongo_connector.get_sonar_reverse_dns_connection()
srdns_collection.remove({'updated': {"$lt": d_minus_2m}})
ip_manager.delete_records_by_date(d_minus_2m)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete")
class UltraDNSZonesInfo(object):
UH = UltraDNSHelper.UltraDNSHelper('get_ultradns_zones_info')
APIH = APIHelper.APIHelper()
DNS_MGR = DNSManager.DNSManager(UH.MC)
def __ultradns_zone_info_response_handler(self, response):
"""
Handles the API response. Incorrect JSON parsing is allowed upto 20 times post which the
script exits.
:param response: Response object
"""
try:
response_data = response.json()
record_sets = response_data['rrSets']
except (ValueError, AttributeError) as err:
if self.UH.incorrect_response_json_allowed > 0:
print('Unable to parse response JSON for zone ' +
self.zone_queried)
self.UH.incorrect_response_json_allowed -= 1
else:
self.APIH.handle_api_error(
'Unable to parse response JSON for 20 zones: ' + repr(err),
self.UH.jobs_manager,
)
else:
for record in record_sets:
dns_info = dict()
# The ownerName could be either the FQDN or a relative domain name.
# In case it is a FQDN it will end in '.'
fqdn = record['ownerName'] + '.' + self.zone_queried
if record['ownerName'].endswith('.'):
fqdn = record['ownerName'][:-1]
dns_info['zone'] = self.zone_queried
dns_info['fqdn'] = fqdn
dns_info['type'] = record['rrtype'].split(' ')[0].lower()
dns_info['status'] = 'unknown'
for dns in record['rdata']:
if dns_info['type'] in ['a', 'ptr']:
try:
if IPAddress(dns).is_private():
continue
except AddrFormatError as err:
print('For ' + fqdn + ' encountered: ' + str(err))
continue
if not (dns_info['type'] in ['soa', 'txt'
]) and dns.endswith('.'):
dns = dns[:-1]
dns_info['value'] = dns
dns_info['created'] = datetime.now()
self.DNS_MGR.insert_record(dns_info.copy(), self.UH.source)
self.UH.set_offset(response_data['resultInfo'])
def __paginated_ultradns_zones_info_request(self):
"""
Makes paginated API calls to UltraDNS. The API is retried 3 times in case of ConnectionError
exception before the script exists. The script exists on encountering HTTPError or any other
RequestException.
In case a 401 is encountered along with the required token expiration message, another login
API is sent with grant_type set as 'refresh_token' to retrieve a valid access token.
"""
url = self.UH.ULTRACONNECT.ZONEINFO.format(
zone_queried=self.zone_queried)
try:
response = self.UH.backoff_api_retry(
url, {
'q': 'kind:RECORDS',
'limit': 2000,
'offset': self.UH.offset,
}, {'authorization': 'Bearer ' + self.UH.access_token})
response.raise_for_status()
except requests.exceptions.HTTPError as herr:
err_msg = json.loads(response.text)['errorMessage']
if response.status_code == 401 and err_msg == self.UH.access_token_expiration_error:
self.UH.login('refresh_token')
self.__paginated_ultradns_zones_info_request()
else:
self.APIH.handle_api_error(herr, self.UH.jobs_manager)
except requests.exceptions.RequestException as err:
self.APIH.handle_api_error(err, self.UH.jobs_manager)
else:
self.__ultradns_zone_info_response_handler(response)
def __get_ultradns_zones_info(self):
"""
Extracts the zone DNS information from UltraDNS in a paginated manner for the UltraDNS zones.
"""
print("Starting: " + str(datetime.now()))
self.UH.jobs_manager.record_job_start()
self.UH.get_previous_zones()
# For querying UltraDNS, we need to query on the exact zones reported
# hence we query for previous_zones.
for zone in self.UH.previous_zones:
self.zone_queried = zone
self.UH.offset = 0
self.__paginated_ultradns_zones_info_request()
while self.UH.offset:
self.__paginated_ultradns_zones_info_request()
# Record status
self.UH.jobs_manager.record_job_complete()
print("Ending: " + str(datetime.now()))
def __init__(self):
self.__get_ultradns_zones_info()
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
if is_running(os.path.basename(__file__)):
logger.warning("Already running...")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
r7 = Rapid7.Rapid7()
mongo_connection = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connection)
ip_manager = IPManager.IPManager(mongo_connection)
rdns_collection = mongo_connection.get_sonar_reverse_dns_connection()
zones = ZoneManager.get_distinct_zones(mongo_connection)
logger.info("Zone length: " + str(len(zones)))
save_directory = "./files/"
parser = argparse.ArgumentParser(
description='Parse Sonar files based on CIDRs.')
parser.add_argument('--sonar_file_type',
required=True,
help='Specify "dns" or "rdns"')
args = parser.parse_args()
check_save_location(save_directory)
# A session is necessary for the multi-step log-in process
s = requests.Session()
if args.sonar_file_type == "rdns":
jobs_manager = JobsManager.JobsManager(mongo_connection,
'get_data_by_cidr_rdns')
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "rdns", jobs_manager)
if html_parser.rdns_url == "":
logger.error("Unknown Error")
jobs_manager.record_job_error()
exit(0)
unzipped_rdns = download_remote_files(logger, s,
html_parser.rdns_url,
save_directory, jobs_manager)
update_rdns(logger, unzipped_rdns, rdns_collection, dns_manager,
ip_manager, zones)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
logger.info("RDNS Complete")
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns":
jobs_manager = JobsManager.JobsManager(mongo_connection,
'get_data_by_cidr_dns')
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.any_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.any_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, dns_manager, ip_manager,
zones)
if html_parser.a_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.a_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, dns_manager, ip_manager,
zones)
if html_parser.aaaa_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.aaaa_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, dns_manager, ip_manager,
zones)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
logger.info("DNS Complete")
jobs_manager.record_job_complete()
else:
logger.error("Unrecognized sonar_file_type option. Exiting...")
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
now = datetime.now()
print("Starting: " + str(now))
azure_connector = AzureConnector.AzureConnector()
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zone_ingestor = ZoneIngestor.ZoneIngestor()
jobs_manager = JobsManager.JobsManager(mongo_connector, 'fetch_azure_dns')
jobs_manager.record_job_start()
current_zones = ZoneManager.get_distinct_zones(mongo_connector)
resource_client = azure_connector.get_resources_client()
resources = []
# The resource list is not currently used.
for item in resource_client.resource_groups.list():
resources.append(item.name)
dns_client = azure_connector.get_dns_client()
zones = dns_client.zones.list()
# The type of records the Azure DNS will let you configure
record_types = {
'A': 'arecords',
'AAAA': 'aaaa_records',
'MX': 'mx_records',
'NS': 'ns_records',
'PTR': 'ptr_records',
'SRV': 'srv_records',
'TXT': 'txt_records',
'CNAME': 'cname_record',
'SOA': 'soa_record'
}
for zone in zones:
print("Zone: " + zone.name)
data = split_id(zone.id)
if zone.zone_type == ZoneType.public:
print(zone.name + " is public:")
if zone.name not in current_zones:
print("Creating zone: " + zone.name)
zone_ingestor.add_zone(zone.name,
"azure:" + data["resourceGroups"])
try:
print("ResourceGroup: " + data["resourceGroups"])
records = dns_client.record_sets.list_all_by_dns_zone(
data["resourceGroups"], zone.name)
for entry in records:
# The record_data id value ends in rtype/rvalue so you must guess the rtype
record_data = split_id(entry.id)
for rtype in record_types:
if rtype in record_data:
results = extract_record_set_value(rtype, entry)
for result in results:
result['zone'] = zone.name
result['created'] = datetime.now()
result['status'] = 'confirmed'
dns_manager.insert_record(
result, "azure:" + data["resourceGroups"])
except:
print("No records found")
jobs_manager.record_job_complete()
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
if is_running(os.path.basename(__file__)):
logger.warning("Already running...")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
parser = argparse.ArgumentParser(
description="Parse Sonar files based on domain zones.")
parser.add_argument(
"--sonar_file_type",
choices=["dns-any", "dns-a", "rdns"],
required=True,
help='Specify "dns-any", "dns-a", or "rdns"',
)
parser.add_argument(
"--database",
choices=["local", "remote"],
required=False,
default="local",
help="Whether to use the local or remote DB",
)
args = parser.parse_args()
r7 = Rapid7.Rapid7()
if args.database == "remote":
mongo_connector = RemoteMongoConnector.RemoteMongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector,
"get_sonar_data_dns")
else:
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zones = ZoneManager.get_distinct_zones(mongo_connector)
save_directory = "./files/"
check_save_location(save_directory)
# A session is necessary for the multi-step log-in process
s = requests.Session()
if args.sonar_file_type == "rdns":
logger.info("Updating RDNS records")
jobs_manager = JobsManager.JobsManager(mongo_connector,
"get_sonar_data_rdns")
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "rdns", jobs_manager)
if html_parser.rdns_url == "":
logger.error("Unknown Error")
jobs_manager.record_job_error()
exit(0)
unzipped_rdns = download_remote_files(logger, s,
html_parser.rdns_url,
save_directory, jobs_manager)
update_rdns(logger, unzipped_rdns, zones, dns_manager,
mongo_connector)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns-any":
logger.info("Updating DNS ANY records")
jobs_manager = JobsManager.JobsManager(mongo_connector,
"get_sonar_data_dns-any")
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.any_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.any_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, dns_manager)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns-a":
logger.info("Updating DNS A, AAAA, and CNAME records")
jobs_manager = JobsManager.JobsManager(mongo_connector,
"get_sonar_data_dns-a")
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.a_url != "":
logger.info("Updating A records")
unzipped_dns = download_remote_files(logger, s,
html_parser.a_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, dns_manager)
if html_parser.aaaa_url != "":
logger.info("Updating AAAA records")
unzipped_dns = download_remote_files(logger, s,
html_parser.aaaa_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, dns_manager)
if html_parser.cname_url != "":
logger.info("Updating CNAME records")
unzipped_dns = download_remote_files(logger, s,
html_parser.cname_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, dns_manager)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
jobs_manager.record_job_complete()
else:
logger.error("Unrecognized sonar_file_type option. Exiting...")
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
mongo_connector = MongoConnector.MongoConnector()
now = datetime.now()
print("Starting: " + str(now))
jobs_manager = JobsManager.JobsManager(mongo_connector, 'owasp_amass')
zones = ZoneManager.get_distinct_zones(mongo_connector)
dns_manager = DNSManager.DNSManager(mongo_connector)
output_dir = "./amass_files/"
arg_parser = argparse.ArgumentParser(
description=
'Run the OWASP Amass tool and store the results in the database.')
arg_parser.add_argument(
'--config_file',
required=False,
help='An optional Amass config file. Otherwise, defaults will be used.'
)
arg_parser.add_argument('--amass_path',
required=True,
help='The path to the amass binary')
arg_parser.add_argument('--output_dir',
default=output_dir,
help="The path where to save Amass files.")
arg_parser.add_argument('--amass_version',
type=int,
default=3,
help='The version of OWASP Amass being used.')
arg_parser.add_argument(
'--sleep',
type=int,
default=5,
help=
'Sleep time in seconds between amass runs so as not to overuse service limits.'
)
args = arg_parser.parse_args()
if not os.path.isfile(args.amass_path):
print("ERROR: Incorrect amass_path argument provided")
exit(1)
if 'config_file' in args and not os.path.isfile(args.config_file):
print("ERROR: Incorrect config_file location")
exit(1)
if 'output_dir' in args:
output_dir = args.output_dir
if not output_dir.endswith("/"):
output_dir = output_dir + "/"
check_save_location(output_dir)
jobs_manager.record_job_start()
for zone in zones:
# Pace out calls to the Amass services
time.sleep(args.sleep)
command_line = []
command_line.append(args.amass_path)
if int(args.amass_version) >= 3:
command_line.append("enum")
if args.config_file:
command_line.append("-config")
command_line.append(args.config_file)
command_line.append("-d")
command_line.append(zone)
command_line.append("-src")
command_line.append("-ip")
command_line.append("-do")
command_line.append(output_dir + zone + "-do.json")
try:
subprocess.check_call(command_line)
except subprocess.CalledProcessError as e:
# Even when there is an error, there will likely still be results.
# We can continue with the data that was collected thus far.
print("ERROR: Amass run exited with a non-zero status: " + str(e))
if os.path.isfile(output_dir + zone + "-do.json"):
output = open(output_dir + zone + "-do.json", "r")
json_data = []
for line in output:
try:
json_data.append(json.loads(line))
except:
print("Amass wrote an incomplete line: " + str(line))
output.close()
for finding in json_data:
if 'type' in finding and finding[
'type'] == 'infrastructure' or finding[
'type'] == 'domain':
# Not currently recording
continue
elif is_tracked_zone(finding['domain'], zones):
record_finding(dns_manager, finding)
else:
# print("Skipping: " + finding['domain'] + " type: " + finding['type'])
pass
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, "sonar_round_two")
google_dns = GoogleDNS.GoogleDNS()
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
results = dns_manager.find_multiple({"type": "cname"}, "sonar_dns")
round_two = []
round_three = []
# Get all the CNAME values from all_dns and append them to round_two
for result in results:
if is_tracked_zone(result["value"], zones):
round_two.append(result["value"])
logger.info("Round two pre-list: " + str(len(round_two)))
dead_dns_collection = mongo_connector.get_dead_dns_connection()
for value in round_two:
is_present = dns_manager.find_count({"fqdn": value}, "sonar_dns")
if is_present == 0:
logger.debug(value + " not found")
time.sleep(1)
result = google_dns.fetch_DNS_records(value)
if result == []:
logger.debug("Unable to resolve")
original_records = dns_manager.find_multiple({"value": value},
"sonar_dns")
for record in original_records:
check = dead_dns_collection.count_documents(
{"fqdn": record["fqdn"]})
if check == 0:
record.pop("_id")
dead_dns_collection.insert(record)
else:
for entry in result:
if is_tracked_zone(entry["fqdn"], zones):
new_record = entry
new_record["status"] = "unconfirmed"
new_record["zone"] = get_fld_from_value(value, "")
new_record["created"] = datetime.now()
if result[0]["type"] == "cname" and is_tracked_zone(
entry["value"], zones):
add_to_list(entry["value"], round_three)
logger.debug("Found: " + value)
if new_record["zone"] != "":
dns_manager.insert_record(new_record, "marinus")
# For each tracked CName result found in the first pass across Sonar DNS
logger.info("Round Three length: " + str(len(round_three)))
for hostname in round_three:
zone = get_fld_from_value(hostname, "")
if zone != None and zone != "":
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
if is_tracked_zone(ip_addr["fqdn"], zones):
record = {"fqdn": ip_addr["fqdn"]}
record["zone"] = get_fld_from_value(
ip_addr["fqdn"], "")
record["created"] = datetime.now()
record["type"] = ip_addr["type"]
record["value"] = ip_addr["value"]
record["status"] = "unconfirmed"
dns_manager.insert_record(new_record, "marinus")
else:
original_record = dns_manager.find_one({"fqdn": hostname},
"marinus")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
logger.debug("Failed IP Lookup for: " + hostname)
else:
logger.debug("Failed match on zone for: " + hostname)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
class UltraDNSZonesInfo(object):
UH = UltraDNSHelper.UltraDNSHelper("get_ultradns_zones_info")
APIH = APIHelper.APIHelper()
DNS_MGR = DNSManager.DNSManager(UH.MC)
_logger = None
def __ultradns_zone_info_response_handler(self, response):
"""
Handles the API response. Incorrect JSON parsing is allowed upto 20 times post which the
script exits.
:param response: Response object
"""
try:
response_data = response.json()
record_sets = response_data["rrSets"]
except (ValueError, AttributeError) as err:
if self.UH.incorrect_response_json_allowed > 0:
self._logger.warning(
"Unable to parse response JSON for zone " +
self.zone_queried)
self.UH.incorrect_response_json_allowed -= 1
else:
self.APIH.handle_api_error(
"Unable to parse response JSON for 20 zones: " + repr(err),
self.UH.jobs_manager,
)
else:
for record in record_sets:
dns_info = dict()
# The ownerName could be either the FQDN or a relative domain name.
# In case it is a FQDN it will end in '.'
fqdn = record["ownerName"] + "." + self.zone_queried
if record["ownerName"].endswith("."):
fqdn = record["ownerName"][:-1]
# A get_root_domain lookup is performed because UDNS supports sub-zones
dns_info["zone"] = ZoneManager.get_root_domain(
self.zone_queried)
dns_info["fqdn"] = fqdn
dns_info["type"] = record["rrtype"].split(" ")[0].lower()
dns_info["status"] = "unknown"
for dns in record["rdata"]:
if dns_info["type"] in ["a", "ptr"]:
try:
if IPAddress(dns).is_private():
continue
except AddrFormatError as err:
self._logger.warning("For " + fqdn +
" encountered: " + str(err))
continue
if not (dns_info["type"] in ["soa", "txt"
]) and dns.endswith("."):
dns = dns[:-1]
dns_info["value"] = dns
dns_info["created"] = datetime.now()
self.DNS_MGR.insert_record(dns_info.copy(), self.UH.source)
self.UH.set_offset(response_data["resultInfo"])
def __paginated_ultradns_zones_info_request(self):
"""
Makes paginated API calls to UltraDNS. The API is retried 3 times in case of ConnectionError
exception before the script exists. The script exists on encountering HTTPError or any other
RequestException.
In case a 401 is encountered along with the required token expiration message, another login
API is sent with grant_type set as 'refresh_token' to retrieve a valid access token.
"""
url = self.UH.ULTRACONNECT.ZONEINFO.format(
zone_queried=self.zone_queried)
try:
response = self.UH.backoff_api_retry(
url,
{
"q": "kind:RECORDS",
"limit": 2000,
"offset": self.UH.offset,
},
{"authorization": "Bearer " + self.UH.access_token},
)
response.raise_for_status()
except requests.exceptions.HTTPError as herr:
message = json.loads(response.text)
if isinstance(message, list):
err_msg = json.loads(response.text)[0]["errorMessage"]
else:
err_msg = json.loads(response.text)["errorMessage"]
if (response.status_code == 401
and err_msg == self.UH.access_token_expiration_error):
self.UH.login("refresh_token")
self.__paginated_ultradns_zones_info_request()
elif response.status_code == 404:
self._logger.warning("ERROR: Could not find data for: " +
str(self.zone_queried))
else:
self.APIH.handle_api_error(herr, self.UH.jobs_manager)
except requests.exceptions.RequestException as err:
self.APIH.handle_api_error(err, self.UH.jobs_manager)
else:
self.__ultradns_zone_info_response_handler(response)
def __get_ultradns_zones_info(self):
"""
Extracts the zone DNS information from UltraDNS in a paginated manner for the UltraDNS zones.
"""
print("Starting: " + str(datetime.now()))
self._logger.info("Starting...")
self.UH.jobs_manager.record_job_start()
self.UH.get_previous_zones()
# For querying UltraDNS, we need to query on the exact zones reported
# hence we query for previous_zones.
for zone in self.UH.previous_zones:
self.zone_queried = zone
self.UH.offset = 0
self.__paginated_ultradns_zones_info_request()
while self.UH.offset:
self.__paginated_ultradns_zones_info_request()
# Record status
self.UH.jobs_manager.record_job_complete()
print("Ending: " + str(datetime.now()))
self._logger.info("Complete.")
def __init__(self):
self._logger = LoggingUtil.create_log(__name__)
self.__get_ultradns_zones_info()
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
'extract_mx_domains')
google_dns = GoogleDNS.GoogleDNS()
jobs_manager.record_job_start()
dns_names = []
round_two = []
zones = ZoneManager.get_distinct_zones(mongo_connector)
# Collect the list of domains from the MX Records
extract_mx_names(dns_names, dns_manager)
input_list = []
# Some MX records point to the third-party domains.
# Therefore, we filter to only the root domains that belong to the tracked company.
print("Pre-filter list: " + str(len(dns_names)))
for hostname in dns_names:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
# Pause to prevent DoS-ing of Google's HTTPS DNS Service
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
if ip_addr['type'] == "cname" and is_tracked_zone(
ip_addr['value'], zones):
add_to_round_two(ip_addr['value'], round_two)
else:
print("Failed IP Lookup for: " + hostname)
else:
print("Failed match on zone for: " + hostname)
dead_dns_collection = mongo_connector.get_dead_dns_connection()
# Some DNS records will be CNAME records pointing to other tracked domains.
# This is a single level recursion to lookup those domains.
print("Round Two list: " + str(len(round_two)))
for hostname in round_two:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
else:
print("Failed IP Lookup for: " + hostname)
original_record = dns_manager.find_one({"fqdn": hostname},
"mx")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
else:
print("Failed match on zone for: " + hostname)
# Record all the results.
dns_manager.remove_by_source("mx")
print("List length: " + str(len(input_list)))
for final_result in input_list:
dns_manager.insert_record(final_result, "mx")
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Connect to the remote databases
mongo_connector = MongoConnector.MongoConnector()
rm_connector = RemoteMongoConnector.RemoteMongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zones = ZoneManager.get_distinct_zones(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, "remote_download")
jobs_manager.record_job_start()
remote_jobs_collection = rm_connector.get_jobs_connection()
# Check the status of the Censys job on the remote database
try:
status = remote_jobs_collection.find_one({"job_name": "censys"})
except:
logger.error("Can not connect to remote database")
jobs_manager.record_job_error()
exit(1)
if (status is not None and "status" in status
and status["status"] != jobs_manager.COMPLETE):
logger.info("Censys scans status is not COMPLETE")
elif (status is not None and "status" in status
and status["status"] == jobs_manager.COMPLETE):
# Get connections to the relevant collections.
censys_collection = mongo_connector.get_zgrab_443_data_connection()
remote_censys_collection = rm_connector.get_zgrab_443_data_connection()
download_censys_scan_info(logger, censys_collection,
remote_censys_collection)
# Tell the remote database that is safe to start processing the next Censys file
remote_jobs_collection.update_one(
{"job_name": "censys"},
{
"$currentDate": {
"updated": True
},
"$set": {
"status": jobs_manager.READY
}
},
)
# Get connections to the relevant HTTPS collections.
zgrab_443_data_collection = mongo_connector.get_zgrab_443_data_connection()
remote_zgrab_443_data_collection = rm_connector.get_zgrab_443_data_connection(
)
download_zgrab_info(logger, zgrab_443_data_collection,
remote_zgrab_443_data_collection)
# Get connections to the relevant HTTP collections.
zgrab_80_data_collection = mongo_connector.get_zgrab_80_data_connection()
remote_zgrab_80_data_collection = rm_connector.get_zgrab_80_data_connection(
)
download_zgrab_info(logger, zgrab_80_data_collection,
remote_zgrab_80_data_collection)
# Get connections to the relevant port collections.
zgrab_port_data_collection = mongo_connector.get_zgrab_port_data_connection(
)
remote_zgrab_port_data_collection = rm_connector.get_zgrab_port_data_connection(
)
download_zgrab_port_info(logger, zgrab_port_data_collection,
remote_zgrab_port_data_collection)
# Download latest whois information
status = remote_jobs_collection.find_one({"job_name": "whois_lookups"})
if status["status"] == jobs_manager.COMPLETE:
whois_collection = mongo_connector.get_whois_connection()
remote_whois_collection = rm_connector.get_whois_connection()
download_whois_data(logger, whois_collection, remote_whois_collection)
remote_jobs_collection.update_one(
{"job_name": "whois"}, {"$set": {
"status": jobs_manager.READY
}})
# Download Amass results
amass_collection = mongo_connector.get_owasp_amass_connection()
remote_amass_collection = rm_connector.get_owasp_amass_connection()
download_amass_data(logger, amass_collection, remote_amass_collection,
dns_manager, zones)
# Download the status of the remote jobs
download_jobs_status(logger, jobs_manager._jobs_collection,
remote_jobs_collection)
# Download remote sonar DNS findings
download_sonar_dns(logger, dns_manager, rm_connector)
# Download remote sonar RDNS findings
download_sonar_rdns(logger, mongo_connector, rm_connector)
# Update the local jobs database to done
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
graphs_collection = mongo_connector.get_graphs_connection()
graphs_data_collection = mongo_connector.get_graphs_data_connection()
graphs_links_collection = mongo_connector.get_graphs_links_connection()
graphs_docs_collection = mongo_connector.get_graphs_docs_connection()
ip_manager = IPManager.IPManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, 'create_graphs2')
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
for zone in zones:
groups = []
graph = nx.Graph()
add_to_list(zone, groups)
graph.add_node(zone,
data_type="tld",
type=0,
depends=[],
dependedOnBy=[],
docs="<h1>Parent</h1>")
find_all_dns_by_zone(graph, zone, groups, dns_manager, ip_manager)
find_srdns_by_zone(graph, zone, groups, mongo_connector, ip_manager)
data = json_graph.node_link_data(graph)
reformat_data(data, zone, groups)
new_data = {}
new_data['directed'] = data['directed']
new_data['graph'] = data['graph']
new_data['multigraph'] = data['multigraph']
new_data['errs'] = []
config = {}
config['title'] = zone + " Network Map"
config['graph'] = {}
config['graph']['linkDistance'] = 150
config['graph']['charge'] = -400
config['graph']['height'] = 800
config['graph']['numColors'] = len(groups)
config['graph']['labelPadding'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['labelMargin'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['ticksWithoutCollisions'] = 50
config['graph_type'] = "tracked_domain"
config['types'] = {}
regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
regx = re.compile(regex_str)
for tgroup in groups:
group = tgroup.replace(REPLACE_CHAR, ".")
data_type = "tpd"
if group in zones:
data_type = "tracked_domain"
elif re.match(regx, group):
data_type = "cidr"
config['types'][tgroup] = {
"short": group,
"long": "A group from the network: " + group,
"data_type": data_type
}
config['constraints'] = []
tmp = int(math.ceil(math.sqrt(len(groups)))) + 1
x = []
y = []
for i in range(1, tmp):
val = round((i * 1.0) / tmp, 2)
x.append(str(val))
y.append(str(val))
x_pos = 0
y_pos = 0
for group in groups:
config['constraints'].append({
"has": {
"type": group
},
"type": "position",
"x": x[x_pos],
"y": y[y_pos]
})
x_pos = x_pos + 1
if x_pos >= len(x):
x_pos = 0
y_pos = y_pos + 1
config['jsonUrl'] = "/api/v1.0/graphs/" + zone
new_data['config'] = config
new_data['created'] = datetime.now()
new_data['zone'] = zone
new_docs_data = {}
new_docs_data['docs'] = {}
new_docs_data['zone'] = zone
new_docs_data['created'] = datetime.now()
new_graph_data = {}
new_graph_data['data'] = {}
for i in range(0, len(data['nodes'])):
new_graph_data['data'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)] = data['nodes'][i]
new_docs_data['docs'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)] = data['nodes'][i]['docs']
del new_graph_data['data'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)]['docs']
new_graph_data['created'] = datetime.now()
new_graph_data['zone'] = zone
new_graph_data['directed'] = data['directed']
new_graph_data['multigraph'] = data['multigraph']
new_graph_data['errs'] = []
new_links_data = {}
new_links_data['links'] = data['links']
new_links_data['created'] = datetime.now()
new_links_data['zone'] = zone
new_links_data['directed'] = data['directed']
new_links_data['multigraph'] = data['multigraph']
new_links_data['errs'] = []
try:
graphs_collection.remove({'zone': zone})
graphs_collection.insert_one(new_data)
graphs_data_collection.remove({'zone': zone})
graphs_data_collection.insert_one(new_graph_data)
graphs_links_collection.remove({'zone': zone})
graphs_links_collection.insert_one(new_links_data)
graphs_docs_collection.remove({'zone': zone})
graphs_docs_collection.insert_one(new_docs_data)
except:
logger.error("ERROR: Can't insert: " + zone)
time.sleep(1)
# Remove last week's old entries
# In theory, shouldn't do anything but being complete
lastweek = datetime.now() - timedelta(days=7)
graphs_collection.remove({'created': {"$lt": lastweek}})
graphs_data_collection.remove({'created': {"$lt": lastweek}})
graphs_links_collection.remove({'created': {"$lt": lastweek}})
graphs_docs_collection.remove({'created': {"$lt": lastweek}})
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete")
def main():
"""
Begin Main...
"""
# The sources for which to remove expired entries
# Infoblox is handled separately
# Sonar RDNS is hard code below in a separate section
# {"source_name": date_difference_in_months}
sources = [{"name": "sonar_dns", "diff": -2},
{"name": "sonar_dns_saved", "diff": -2},
{"name": "ssl", "diff": -2},
{"name": "ssl_saved", "diff": -2},
{"name": "virustotal", "diff": -2},
{"name": "virustotal_saved", "diff": -2},
{"name": "UltraDNS", "diff": -2},
{"name": "UltraDNS_saved", "diff": -2},
{"name": "skms", "diff": -2},
{"name": "skms_saved", "diff": -2},
{"name": "marinus", "diff": -2},
{"name": "marinus_saved", "diff": -2},
{"name": "mx", "diff": -2},
{"name": "mx_saved", "diff": -2},
{"name": "common_crawl", "diff": -4},
{"name": "common_crawl_saved", "diff": -4}]
now = datetime.now()
print ("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
dns_manager = DNSManager.DNSManager(mongo_connector)
GDNS = GoogleDNS.GoogleDNS()
zones = ZoneManager.get_distinct_zones(mongo_connector)
jobs_collection = mongo_connector.get_jobs_connection()
# Get the date for today minus two months
d_minus_2m = monthdelta(datetime.now(), -2)
print("Removing SRDNS as of: " + str(d_minus_2m))
# Remove the old records
srdns_collection = mongo_connector.get_sonar_reverse_dns_connection()
srdns_collection.remove({'updated': {"$lt": d_minus_2m}})
# Before completely removing old entries, make an attempt to see if they are still valid.
# Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source.
# Rather than throw away valid information, this will archive it.
for entry in sources:
removal_date = monthdelta(datetime.now(), entry['diff'])
source = entry['name']
print("Removing " + source + " as of: " + str(removal_date))
last_domain = ""
results = all_dns_collection.find({'sources': {"$size": 1}, 'sources.source': source, 'sources.updated': {"$lt": removal_date}})
for result in results:
if result['fqdn'] != last_domain:
last_domain = result['fqdn']
dns_result = GDNS.fetch_DNS_records(result['fqdn'], GDNS.DNS_TYPES[result['type']])
if dns_result != []:
for dns_entry in dns_result:
if is_tracked_zone(dns_entry['fqdn'], zones):
new_entry={}
new_entry['updated'] = datetime.now()
new_entry['zone'] = result['zone']
new_entry['fqdn'] = dns_entry['fqdn']
new_entry['created'] = result['created']
new_entry['value'] = dns_entry['value']
new_entry['type'] = dns_entry['type']
new_entry['status'] = 'confirmed'
if 'sonar_timestamp' in result:
new_entry['sonar_timestamp'] = result['sonar_timestamp']
if source.endswith("_saved"):
dns_manager.insert_record(new_entry, source)
else:
dns_manager.insert_record(new_entry, source + "_saved")
dns_manager.remove_all_by_source_and_date(source, entry['diff'])
# Record status
jobs_collection.update_one({'job_name': 'remove_expired_entries'},
{'$currentDate': {"updated": True},
"$set": {'status': 'COMPLETE'}})
now = datetime.now()
print("Complete: " + str(now))
import json
import logging
import os
import requests
import subprocess
import sys
import time
from datetime import datetime
from libs3 import DNSManager, MongoConnector, Rapid7, JobsManager, GoogleDNS
from libs3.ZoneManager import ZoneManager
from libs3.LoggingUtil import LoggingUtil
mongo_connector = MongoConnector.MongoConnector()
global_dns_manager = DNSManager.DNSManager(mongo_connector)
def is_running(process):
"""
Is the provided process name is currently running?
"""
proc_list = subprocess.Popen(["pgrep", "-f", process],
stdout=subprocess.PIPE)
for proc in proc_list.stdout:
if proc.decode('utf-8').rstrip() != str(
os.getpid()) and proc.decode('utf-8').rstrip() != str(
os.getppid()):
return True
return False
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
google_dns = GoogleDNS.GoogleDNS()
jobs_manager = JobsManager.JobsManager(mongo_connector,
'extract_ssl_domains')
jobs_manager.record_job_start()
parser = argparse.ArgumentParser(
description='Search TLS certificates for additional DNS names')
parser.add_argument('--zgrab_version',
default=2,
type=int,
choices=[1, 2],
metavar="version",
help='The version of ZGrab used to collect data')
args = parser.parse_args()
dns_names = []
round_two = []
zones = ZoneManager.get_distinct_zones(mongo_connector)
# Collect the list of domains from the SSL Certificates
extract_ct_certificate_names(dns_names, mongo_connector)
# extract_censys_certificate_names(dns_names, mongo_connector)
if args.zgrab_version == 1:
extract_zgrab_certificate_names(logger, dns_names, mongo_connector)
else:
extract_zgrab2_certificate_names(logger, dns_names, mongo_connector)
input_list = []
# Some SSL certificates are for multiple domains.
# The tracked company may not own all domains.
# Therefore, we filter to only the root domains that belong to the tracked company.
logger.info("Pre-filter list: " + str(len(dns_names)))
for hostname in dns_names:
if not hostname.startswith("*"):
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
# Pause to prevent DoS-ing of Google's HTTPS DNS Service
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
if ip_addr['type'] == "cname" and is_tracked_zone(
ip_addr['value'], zones):
add_to_round_two(ip_addr['value'], round_two)
else:
logger.warning("Failed IP Lookup for: " + hostname)
else:
logger.warning("Failed match on zone for: " + hostname)
else:
logger.warning("Skipping wildcard: " + hostname)
dead_dns_collection = mongo_connector.get_dead_dns_connection()
# Some DNS records will be CNAME records pointing to other tracked domains.
# This is a single level recursion to lookup those domains.
logger.info("Round Two list: " + str(len(round_two)))
for hostname in round_two:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
else:
logger.warning("Failed IP Lookup for: " + hostname)
original_record = dns_manager.find_one({"fqdn": hostname},
"ssl")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
else:
logger.warning("Failed match on zone for: " + hostname)
# Record all the results.
dns_manager.remove_by_source("ssl")
logger.info("List length: " + str(len(input_list)))
for final_result in input_list:
dns_manager.insert_record(final_result, "ssl")
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete")
class InfobloxDNSManager(object):
# Make database connections
MC = MongoConnector.MongoConnector()
zone_collection = MC.get_zone_connection()
APIH = APIHelper.APIHelper()
IH = InfobloxHelper.InfobloxHelper()
DNS_MGR = DNSManager.DNSManager(MC)
next_page_id = None
zone_queried = None
record_type = None
iblox_collection = None
dns_value_mapper = {
"mx": "mail_exchanger",
"txt": "text",
"a": "ipv4addr",
"cname": "canonical",
"aaaa": "ipv6addr",
}
_logger = None
def _log(self):
"""
Get the log
"""
return logging.getLogger(__name__)
def __get_record_type_url(self):
"""
Returns the url to be queried at infoblox to return the DNS information.
Paging information is appended to the URL as per the condition satisfied.
:return: string: URL to be queried
"""
paging_info = self.IH.get_pagination_params(self.next_page_id)
url = self.IH.get_infoblox_base_url(
self.zone_queried,
self.record_type,
).format(
return_fields="&_return_fields%2B=zone",
paging_info=paging_info,
)
return url
def __get_previous_records(self):
"""
Retrieve the current data related to the zone. This is evaluated against the data that we
receive in the latest script run to determine stale records.
The data is stored as a list of _ref.
"""
self.previous_records = self.iblox_collection.distinct(
"_ref", {"zone": self.zone_queried})
def __insert_dns_information(self, dns_information):
"""
Inserts the DNS information into the all_dns collection. For the 'host' records, iterate over
all the ipv4addrs mentioned to get the data.
:param dns_information: DNS data for the zone and 'record_type'
"""
del dns_information["_ref"]
del dns_information["view"]
del dns_information["infoblox_zone"]
if self.record_type == "host":
# In order to resolve multiple ipv4addrs
for ipv4 in dns_information["ipv4addrs"]:
dns_info = dict()
dns_info["zone"] = dns_information["zone"]
dns_info["type"] = "a"
dns_info["value"] = ipv4["ipv4addr"]
dns_info["fqdn"] = ipv4["host"]
dns_info["status"] = "unknown"
dns_info["created"] = datetime.now()
self.DNS_MGR.insert_record(dns_info, "infoblox-host")
else:
# Removing the 'preference' key from the 'mx' records
if self.record_type == "mx":
del dns_information["preference"]
dns_information["value"] = dns_information[self.dns_value_mapper[
self.record_type]]
del dns_information[self.dns_value_mapper[self.record_type]]
dns_information["fqdn"] = dns_information["name"]
del dns_information["name"]
dns_information["type"] = self.record_type
dns_information["status"] = "unknown"
dns_information["created"] = datetime.now()
self.DNS_MGR.insert_record(dns_information,
"infoblox-" + self.record_type)
def __insert_records(self, insert_object):
"""
Inserts/Updates the dns information in the database. '_ref' uniquely identifies the
resource. The data is injected into the individual collections belonging to the
record_type and also into the all_dns collection.
:param insert_object: Dictionary containing the details of the resource.
"""
dns_information = insert_object.copy()
if not insert_object["_ref"] in self.previous_records:
insert_object["created"] = datetime.now()
insert_object["updated"] = datetime.now()
self.iblox_collection.insert(insert_object)
else:
self.previous_records.remove(insert_object["_ref"])
insert_object["updated"] = datetime.now()
self.iblox_collection.update_one({"_ref": insert_object["_ref"]},
{"$set": insert_object})
# Update DNS Information.
self.__insert_dns_information(dns_information)
def __infoblox_response_handler(self, response):
"""
Handles the API response. Incorrect JSON parsing is allowed upto 20 times post which the
script exits. "next_page_id" holds the pagination information.
:param response: Response object
"""
try:
response_data = response.json()
response_result = response_data["result"]
except (ValueError, AttributeError) as err:
if self.incorrect_response_json_allowed > 0:
self._logger.warning(
"Unable to parse response JSON for zone " +
self.zone_queried)
self.incorrect_response_json_allowed -= 1
else:
self.APIH.handle_api_error(
"Unable to parse response JSON for 20 zones: " + repr(err),
"get_iblox_" + self.record_type.lower(),
)
else:
# Add the zone parameter to each record and insert
for entry in response_result:
entry["infoblox_zone"] = entry["zone"]
entry["zone"] = self.zone_queried
self.__insert_records(entry)
if "next_page_id" in response_data:
self.next_page_id = response_data["next_page_id"]
@backoff.on_exception(
backoff.expo,
requests.exceptions.ConnectionError,
max_tries=4,
factor=10,
on_backoff=APIH.connection_error_retry,
)
def __backoff_api_retry(self):
"""
Makes API calls to Infoblox with exponential retry capabilities using 'backoff'. The API is
retried 3 times in case of ConnectionError exception before the script exists.
:return:
"""
return requests.get(
(self.__get_record_type_url()),
auth=HTTPBasicAuth(self.IH.IBLOX_UNAME, self.IH.IBLOX_PASSWD),
verify="/etc/ssl/certs/ca-bundle.crt",
timeout=120,
)
def __infoblox_paginated_request(self):
"""
Makes paginated API calls to Infoblox. The API is retried 3 times in case of ConnectionError
exception before the script exists. The script exists on encountering HTTPError or any other
RequestException. On success, the next_page_id is set to None for the next API call.
"""
try:
response = self.__backoff_api_retry()
response.raise_for_status()
except requests.exceptions.HTTPError as herr:
self.APIH.handle_api_error(herr,
"get_iblox_" + self.record_type.lower())
except requests.exceptions.RequestException as err:
self.APIH.handle_api_error(err,
"get_iblox_" + self.record_type.lower())
else:
self.next_page_id = None
self.__infoblox_response_handler(response)
def get_infoblox_dns(self):
"""
Extracts the zones from the zone collection to query Infoblox. The API calls continue to be made
for the zone till the next_page_id is set to None indicating no new results to be fetched.
Post the retrieval of all the data, the archaic data for a zone is purged.
"""
zones = ZoneManager.get_zones_by_source(self.MC, "Infoblox")
for zone in zones:
self.zone_queried = zone
self.next_page_id = None
self.__get_previous_records()
self.__infoblox_paginated_request()
while self.next_page_id:
self.__infoblox_paginated_request()
self.IH.clean_collection(self.previous_records,
self.iblox_collection)
def __init__(self, record_type):
self.record_type = record_type
self.iblox_collection = self.MC.__getattribute__(
self.IH.IBLOX_COLLECTIONS[self.record_type])()
self.incorrect_response_json_allowed = self.APIH.INCORRECT_RESPONSE_JSON_ALLOWED
self._logger = self._log()
def main():
"""
Begin main...
"""
parser = argparse.ArgumentParser(
description='Search the Common Crawl graph dataset for new domains')
parser.add_argument('--url',
metavar="URL",
help='The URL for the latest vertices file')
args = parser.parse_args()
if args.url != None:
CURRENT_FILE_LIST = args.url
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_collection = mongo_connector.get_jobs_connection()
reversed_zones = ZoneManager.get_reversed_zones(mongo_connector)
alphabet = list(string.digits + string.ascii_lowercase)
# Create a dictionary of the zones grouped by their first letter
# This will allow us to reduce the number of comparisons in the alphabetized CC files.
grouped_zones = {}
for letter in alphabet:
grouped_zones[letter] = []
for zone in reversed_zones:
first_letter = zone[0]
grouped_zones[first_letter].append(zone)
compressed_download_list = download_file(CURRENT_FILE_LIST)
subprocess.check_call(["gunzip", "-f", compressed_download_list])
download_list = compressed_download_list.split(".")[:-1]
list_file = ".".join(download_list)
vertices_file_entries = open(list_file, "r")
for entry in vertices_file_entries:
# Download file
vert_file_url = "http://commoncrawl.s3.amazonaws.com/" + entry.rstrip(
"\n")
compressed_vertices_file = download_file(vert_file_url)
# Decompress file
subprocess.check_call(["gunzip", "-f", compressed_vertices_file])
vertices_list = compressed_vertices_file.split(".")[:-1]
vertices_file = ".".join(vertices_list)
# Get the first and last line of the file
(first_line, last_line) = get_first_and_last_line(vertices_file)
# Get the first and last domain
parts = first_line.split("\t")
first_domain = parts[1].rstrip("\n")
first_char = first_domain[0]
parts = last_line.split("\t")
last_domain = parts[1].rstrip("\n")
last_char = last_domain[0]
# Get the list of zones relevant to that range
searchable_zones = get_zone_sublist(first_char, last_char,
grouped_zones)
# Parse file and insert matches
parse_file(vertices_file, searchable_zones, dns_manager)
subprocess.check_call(["rm", vertices_file])
# Remove all entries more than two months old
# Note: This commented out because Common Crawl graph data is not additive.
# dns_manager.remove_all_by_source_and_date("common_crawl", -4)
jobs_collection.update_one({'job_name': 'common_crawl_graph'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
"extract_vt_domains")
google_dns = GoogleDNS.GoogleDNS()
jobs_manager.record_job_start()
round_two = []
zones = ZoneManager.get_distinct_zones(mongo_connector)
vt_collection = mongo_connector.get_virustotal_connection()
vt_results = vt_collection.find({
"subdomains": {
"$exists": True
}
}, {
"zone": 1,
"subdomains": 1
}).batch_size(30)
input_list = []
# For each result found in the first pass across VirusTotal
for result in vt_results:
# Pause to prevent DoS-ing of Google's HTTPS DNS Service
time.sleep(1)
for hostname in result["subdomains"]:
ips = google_dns.fetch_DNS_records(hostname)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr["fqdn"], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr["fqdn"]}
record["zone"] = temp_zone
record["created"] = datetime.now()
record["type"] = ip_addr["type"]
record["value"] = ip_addr["value"]
record["status"] = "unknown"
input_list.append(record)
if ip_addr["type"] == "cname" and is_tracked_zone(
ip_addr["value"], zones):
add_to_list(ip_addr["value"], round_two)
else:
logger.warning("Failed IP Lookup for: " + hostname)
dead_dns_collection = mongo_connector.get_dead_dns_connection()
# For each tracked CName result found in the first pass across VirusTotal
logger.info("Round Two length: " + str(len(round_two)))
for hostname in round_two:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr["fqdn"], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr["fqdn"]}
record["zone"] = temp_zone
record["created"] = datetime.now()
record["type"] = ip_addr["type"]
record["value"] = ip_addr["value"]
record["status"] = "unknown"
input_list.append(record)
else:
original_record = dns_manager.find_one({"fqdn": hostname},
"virustotal")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
logger.warning("Failed IP Lookup for: " + hostname)
else:
logger.warning("Failed match on zone for: " + hostname)
# Update the database
dns_manager.remove_by_source("virustotal")
logger.info("List length: " + str(len(input_list)))
for final_result in input_list:
dns_manager.insert_record(final_result, "virustotal")
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
# The sources for which to remove expired entries
# Infoblox is handled separately
# {"source_name": date_difference_in_months}
sources = [{
"name": "sonar_dns",
"diff": -2
}, {
"name": "sonar_dns_saved",
"diff": -2
}, {
"name": "sonar_rdns",
"diff": -2
}, {
"name": "sonar_rdns_saved",
"diff": -2
}, {
"name": "ssl",
"diff": -2
}, {
"name": "ssl_saved",
"diff": -2
}, {
"name": "virustotal",
"diff": -2
}, {
"name": "virustotal_saved",
"diff": -2
}, {
"name": "UltraDNS",
"diff": -2
}, {
"name": "UltraDNS_saved",
"diff": -2
}, {
"name": "marinus",
"diff": -2
}, {
"name": "marinus_saved",
"diff": -2
}, {
"name": "mx",
"diff": -2
}, {
"name": "mx_saved",
"diff": -2
}, {
"name": "common_crawl",
"diff": -4
}, {
"name": "common_crawl_saved",
"diff": -4
}]
amass_diff = -2
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
dns_manager = DNSManager.DNSManager(mongo_connector)
GDNS = GoogleDNS.GoogleDNS()
ip_manager = IPManager.IPManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
'remove_expired_entries')
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
# Get the date for today minus two months
d_minus_2m = monthdelta(datetime.now(), -2)
print("Removing SRDNS as of: " + str(d_minus_2m))
# Remove the old records
srdns_collection = mongo_connector.get_sonar_reverse_dns_connection()
srdns_collection.remove({'updated': {"$lt": d_minus_2m}})
ip_manager.delete_records_by_date(d_minus_2m)
# Before completely removing old entries, make an attempt to see if they are still valid.
# Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source.
# Rather than throw away valid information, this will archive it.
for entry in sources:
removal_date = monthdelta(datetime.now(), entry['diff'])
source = entry['name']
print("Removing " + source + " as of: " + str(removal_date))
last_domain = ""
results = all_dns_collection.find({
'sources': {
"$size": 1
},
'sources.source': source,
'sources.updated': {
"$lt": removal_date
}
})
for result in results:
if result['fqdn'] != last_domain:
last_domain = result['fqdn']
lookup_int = get_lookup_int(result, GDNS)
dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int)
if dns_result != []:
insert_current_results(dns_result, dns_manager, zones,
result, source)
dns_manager.remove_all_by_source_and_date(source, entry['diff'])
# Process amass entries
temp_sources = mongo_connector.perform_distinct(all_dns_collection,
'sources.source')
amass_sources = []
for entry in temp_sources:
if entry.startswith("amass:"):
amass_sources.append(entry)
for source in amass_sources:
removal_date = monthdelta(datetime.now(), amass_diff)
print("Removing " + source + " as of: " + str(removal_date))
last_domain = ""
results = mongo_connector.perform_find(
all_dns_collection, {
'sources': {
"$size": 1
},
'sources.source': source,
'sources.updated': {
"$lt": removal_date
}
})
for result in results:
if result['fqdn'] != last_domain:
last_domain = result['fqdn']
lookup_int = get_lookup_int(result, GDNS)
dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int)
if dns_result != []:
insert_current_results(dns_result, dns_manager, zones,
result, source)
dns_manager.remove_all_by_source_and_date(source, amass_diff)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, 'sonar_round_two')
google_dns = GoogleDNS.GoogleDNS()
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
results = dns_manager.find_multiple({'type': 'cname'}, "sonar_dns")
round_two = []
round_three = []
# Get all the CNAME values from all_dns and append them to round_two
for result in results:
if is_tracked_zone(result['value'], zones):
round_two.append(result['value'])
print("Round two pre-list: " + str(len(round_two)))
dead_dns_collection = mongo_connector.get_dead_dns_connection()
for value in round_two:
is_present = dns_manager.find_count({'fqdn': value}, "sonar_dns")
if is_present == 0:
print(value + " not found")
time.sleep(1)
result = google_dns.fetch_DNS_records(value)
if result == []:
print("Unable to resolve")
original_records = dns_manager.find_multiple({"value": value},
"sonar_dns")
for record in original_records:
check = dead_dns_collection.find({
'fqdn': record['fqdn']
}).count()
if check == 0:
record.pop("_id")
dead_dns_collection.insert(record)
else:
for entry in result:
if is_tracked_zone(entry['fqdn'], zones):
new_record = entry
new_record['status'] = 'unconfirmed'
new_record['zone'] = get_fld_from_value(value, '')
new_record['created'] = datetime.now()
if result[0]['type'] == "cname" and is_tracked_zone(
entry['value'], zones):
add_to_list(entry['value'], round_three)
print("Found: " + value)
if new_record['zone'] != '':
dns_manager.insert_record(new_record, "marinus")
# For each tracked CName result found in the first pass across Sonar DNS
print("Round Three length: " + str(len(round_three)))
for hostname in round_three:
zone = get_fld_from_value(hostname, '')
if zone != None and zone != '':
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
if is_tracked_zone(ip_addr['fqdn'], zones):
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = get_fld_from_value(
ip_addr['fqdn'], '')
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unconfirmed'
dns_manager.insert_record(new_record, "marinus")
else:
original_record = dns_manager.find_one({"fqdn": hostname},
"marinus")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
print("Failed IP Lookup for: " + hostname)
else:
print("Failed match on zone for: " + hostname)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
