def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Make database connections
mongo_connector = MongoConnector.MongoConnector()
ct_collection = mongo_connector.get_certificate_transparency_connection()
config_collection = mongo_connector.get_config_connection()
x509parser = X509Parser.X509Parser()
zones = ZoneManager.get_distinct_zones(mongo_connector)
result = config_collection.find_one({}, {"SSL_Orgs": 1, "_id": 0})
ssl_orgs = result["SSL_Orgs"]
# Defaults
save_location = "/mnt/workspace/"
download_method = "dbAndSave"
save_type = "PEM"
parser = argparse.ArgumentParser(
description="Download certificate information from the provide CT Log."
)
parser.add_argument(
"--log_source",
required=True,
help=
"Indicates which log to query based on values in the x509Parser library",
)
parser.add_argument(
"--include_precerts",
action="store_true",
help="Include pre-certificates which are not finalized",
)
parser.add_argument(
"--download_methods",
choices=["dbAndSave", "dbOnly"],
default=download_method,
help=
"Indicates whether to download the raw files or just save to the database",
)
parser.add_argument(
"--starting_index",
required=False,
default=-1,
type=int,
help="Force the script to start at specific index within the log.",
)
parser.add_argument(
"--cert_save_location",
required=False,
default=save_location,
help=
"Indicates where to save the certificates on disk when choosing dbAndSave",
)
parser.add_argument(
"--save_type",
choices=["PEM", "ASN1"],
default=save_type,
help="Indicates which format to use for the data. The default is PEM",
)
args = parser.parse_args()
source = args.log_source
try:
ct_log_map = x509parser.CT_LOG_MAP[source]
except:
logger.error("ERROR: UNKNOWN LOG SOURCE: " + source)
exit(1)
if args.cert_save_location:
save_location = args.cert_save_location
if not save_location.endswith("/"):
save_location = save_location + "/"
if args.download_methods:
download_method = args.download_methods
check_save_location(save_location, source)
if args.save_type:
save_type = args.save_type
jobs_manager = JobsManager.JobsManager(mongo_connector, "ct_log-" + source)
jobs_manager.record_job_start()
if args.starting_index == -1:
starting_index = fetch_starting_index(ct_collection, source)
else:
starting_index = args.starting_index
logger.info("Starting Index: " + str(starting_index))
sth_data = fetch_sth(logger, "https://" + ct_log_map["url"], jobs_manager)
logger.info("Tree size: " + str(sth_data["tree_size"]))
current_index = starting_index
while current_index < sth_data["tree_size"]:
ending_index = current_index + 256
if ending_index > sth_data["tree_size"]:
ending_index = sth_data["tree_size"]
logger.debug("Checking from index: " + str(current_index) +
" to index " + str(ending_index))
certs = fetch_certificate_batch(
logger,
"https://" + ct_log_map["url"],
current_index,
ending_index,
jobs_manager,
)
for entry in certs["entries"]:
der_cert, cert_type = get_cert_from_leaf(logger,
entry["leaf_input"])
if der_cert is None and cert_type == 1 and not args.include_precerts:
current_index = current_index + 1
continue
elif der_cert is None and cert_type == 0:
current_index = current_index + 1
continue
elif der_cert is None and cert_type == 1:
der_cert = get_cert_from_extra_data(entry["extra_data"])
cert = x509parser.parse_data(der_cert, source)
if cert is None:
logger.warning("Skipping certificate index: " +
str(current_index))
current_index = current_index + 1
continue
if cert_type == 1:
cert["ct_log_type"] = "PRE-CERTIFICATE"
else:
cert["ct_log_type"] = "CERTIFICATE"
cert_zones = check_zone_relevancy(cert, zones)
if check_org_relevancy(cert, ssl_orgs) or cert_zones != []:
cert[source + "_id"] = current_index
cert["zones"] = cert_zones
logger.info("Adding " + source + " id: " + str(current_index) +
" SHA256: " + cert["fingerprint_sha256"])
insert_certificate(cert, source, ct_collection, cert_zones)
if download_method == "dbAndSave":
write_file(logger, cert, save_location, save_type, source)
current_index = current_index + 1
# Set isExpired for any entries that have recently expired.
ct_collection.update_many(
{
"not_after": {
"$lt": datetime.utcnow()
},
"isExpired": False
},
{"$set": {
"isExpired": True
}},
)
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
# Make database connections
mongo_connector = MongoConnector.MongoConnector()
ct_collection = mongo_connector.get_certificate_transparency_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector, "facebook_certs")
jobs_manager.record_job_start()
file_path = "/mnt/workspace/ct_facebook/"
fb_connector = FacebookConnector.FacebookConnector()
access_token = fb_connector.get_facebook_access_token()
zones = ZoneManager.get_distinct_zones(mongo_connector)
x509_parser = X509Parser.X509Parser()
parser = argparse.ArgumentParser(
description='Download DNS and/or certificate information from crt.sh.')
parser.add_argument(
'--fetch_cert_records',
choices=['dbAndSave', 'dbOnly'],
default="dbAndSave",
help=
'Indicates whether to download the raw files or just record in the database'
)
parser.add_argument(
'--cert_save_location',
required=False,
default=file_path,
help=
'Indicates where to save the certificates on disk when choosing dbAndSave'
)
args = parser.parse_args()
check_save_location(args.cert_save_location)
save_location = args.cert_save_location
if not save_location.endswith("/"):
save_location = save_location + "/"
for zone in zones:
time.sleep(15)
results = fetch_domain(fb_connector, access_token, zone)
if results is None:
print("ERROR looking up: " + zone)
continue
print(zone + ": " + str(len(results)))
for result in results:
if args.fetch_cert_records == "dbAndSave":
cert_f = open(
save_location + zone + "_" + result['id'] + ".pem", "w")
cert_f.write(result['certificate_pem'])
cert_f.close()
cert = x509_parser.parse_data(result['certificate_pem'],
"facebook")
cert['facebook_id'] = result['id']
if ct_collection.find({
'fingerprint_sha256':
cert['fingerprint_sha256']
}).count() == 0:
ct_collection.insert(cert)
else:
ct_collection.update(
{'fingerprint_sha256': cert['fingerprint_sha256']},
{"$addToSet": {
'zones': zone
}})
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
google_dns = GoogleDNS.GoogleDNS()
jobs_manager = JobsManager.JobsManager(mongo_connector,
'extract_ssl_domains')
jobs_manager.record_job_start()
parser = argparse.ArgumentParser(
description='Search TLS certificates for additional DNS names')
parser.add_argument('--zgrab_version',
default=2,
type=int,
choices=[1, 2],
metavar="version",
help='The version of ZGrab used to collect data')
args = parser.parse_args()
dns_names = []
round_two = []
zones = ZoneManager.get_distinct_zones(mongo_connector)
# Collect the list of domains from the SSL Certificates
extract_ct_certificate_names(dns_names, mongo_connector)
# extract_censys_certificate_names(dns_names, mongo_connector)
if args.zgrab_version == 1:
extract_zgrab_certificate_names(dns_names, mongo_connector)
else:
extract_zgrab2_certificate_names(dns_names, mongo_connector)
input_list = []
# Some SSL certificates are for multiple domains.
# The tracked company may not own all domains.
# Therefore, we filter to only the root domains that belong to the tracked company.
print("Pre-filter list: " + str(len(dns_names)))
for hostname in dns_names:
if not hostname.startswith("*"):
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
# Pause to prevent DoS-ing of Google's HTTPS DNS Service
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
if ip_addr['type'] == "cname" and is_tracked_zone(
ip_addr['value'], zones):
add_to_round_two(ip_addr['value'], round_two)
else:
print("Failed IP Lookup for: " + hostname)
else:
print("Failed match on zone for: " + hostname)
else:
print("Skipping wildcard: " + hostname)
dead_dns_collection = mongo_connector.get_dead_dns_connection()
# Some DNS records will be CNAME records pointing to other tracked domains.
# This is a single level recursion to lookup those domains.
print("Round Two list: " + str(len(round_two)))
for hostname in round_two:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
else:
print("Failed IP Lookup for: " + hostname)
original_record = dns_manager.find_one({"fqdn": hostname},
"ssl")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
else:
print("Failed match on zone for: " + hostname)
# Record all the results.
dns_manager.remove_by_source("ssl")
print("List length: " + str(len(input_list)))
for final_result in input_list:
dns_manager.insert_record(final_result, "ssl")
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin Main
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Set up all the database connections
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
'create_netaddr_graphs')
jobs_manager.record_job_start()
# Get the list of the all Class C's in Marinus
groups = []
create_list_of_cidrs(groups, mongo_connector, dns_manager)
# Create a separate copy of the class C list since groups will be modified later
cidr_list = groups + []
# Create the stats on the network data
group_data = create_network_data_sets(groups, mongo_connector)
logger.info("Number of Tracked Class C's: " +
str(group_data['tracked_count']))
logger.info("Number of AWS Class C's: " + str(group_data['aws_count']))
logger.info("Number of Azure Class C's: " + str(group_data['azure_count']))
logger.info("Number of Akamai Class C's: " +
str(group_data['akamai_count']))
logger.info("Number of Class C's: " + str(len(groups)))
# Get the current list of zones
zones = ZoneManager.get_distinct_zones(mongo_connector)
# For each Class C that was identified in Marinus...
for tcidr in cidr_list:
cidr = tcidr.replace(REPLACE_CHAR, ".")
groups = []
graph = nx.Graph()
add_to_list(cidr, groups)
graph.add_node(cidr,
data_type="class_c",
type=0,
depends=[],
dependedOnBy=[],
docs="<h1>Parent</h1>")
find_all_dns_by_zone(graph, cidr, groups, dns_manager)
find_srdns_by_zone(graph, cidr, groups, mongo_connector)
data = json_graph.node_link_data(graph)
reformat_data(data, cidr, groups)
new_data = {}
new_data['directed'] = data['directed']
new_data['graph'] = data['graph']
new_data['multigraph'] = data['multigraph']
new_data['errs'] = []
new_data['links'] = data['links']
new_data['data'] = {}
for i in range(0, len(data['nodes'])):
new_data['data'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)] = data['nodes'][i]
config = {}
config['title'] = cidr + " Network Map"
config['graph'] = {}
config['graph']['linkDistance'] = 150
config['graph']['charge'] = -400
config['graph']['height'] = 800
config['graph']['numColors'] = len(groups)
config['graph']['labelPadding'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['labelMargin'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['ticksWithoutCollisions'] = 50
config['graph_type'] = "cidr"
config['types'] = {}
regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
regx = re.compile(regex_str)
for tgroup in groups:
group = tgroup.replace(REPLACE_CHAR, ".")
data_type = "tpd"
if group in zones:
data_type = "tracked_domain"
elif re.match(regx, group):
data_type = "cidr"
config['types'][tgroup] = {
"short": group,
"long": "A group from the network: " + group,
"data_type": data_type
}
config['constraints'] = []
tmp = int(math.ceil(math.sqrt(len(groups)))) + 1
x = []
y = []
for i in range(1, tmp):
val = round((i * 1.0) / tmp, 2)
x.append(str(val))
y.append(str(val))
x_pos = 0
y_pos = 0
for group in groups:
config['constraints'].append({
"has": {
"type": group
},
"type": "position",
"x": x[x_pos],
"y": y[y_pos]
})
x_pos = x_pos + 1
if x_pos >= len(x):
x_pos = 0
y_pos = y_pos + 1
config['jsonUrl'] = "/api/v1.0/cidr_graphs/" + cidr
new_data['config'] = config
new_data['created'] = datetime.now()
new_data['zone'] = cidr
cidr_graphs_collection = mongo_connector.get_cidr_graphs_connection()
cidr_graphs_collection.remove({'zone': cidr})
cidr_graphs_collection.insert_one(new_data)
time.sleep(1)
# Remove last week's old entries
lastweek = datetime.now() - timedelta(days=7)
cidr_graphs_collection.remove({'created': {"$lt": lastweek}})
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
global global_exit_flag
global global_zgrab_path
parser = argparse.ArgumentParser(
description='Launch zgrab against IPs using port 80 or 443.')
parser.add_argument('-p',
choices=['443', '80'],
metavar="port",
help='The web port: 80 or 443')
parser.add_argument('-t',
default=5,
type=int,
metavar="threadCount",
help='The number of threads')
parser.add_argument('--zgrab_path',
default=global_zgrab_path,
metavar='zgrabVersion',
help='The version of ZGrab to use')
args = parser.parse_args()
if args.p == None:
print("A port value (80 or 443) must be provided.")
exit(0)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
now = datetime.now()
print(str(now) + ": I am already running! Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
rm_connector = RemoteMongoConnector.RemoteMongoConnector()
all_dns_collection = rm_connector.get_all_dns_connection()
ip_manager = IPManager.IPManager(rm_connector, True)
jobs_manager = JobsManager.JobsManager(rm_connector,
"zgrab_http_ip-" + args.p)
jobs_manager.record_job_start()
zones_struct = {}
zones_struct['zones'] = ZoneManager.get_distinct_zones(rm_connector)
# Not pretty but cleaner than previous method
zones_struct['ip_manager'] = ip_manager
(ips, ip_context) = get_ips(ip_manager, all_dns_collection)
print("Got IPs: " + str(len(ips)))
zones_struct['ip_context'] = ip_context
if args.p == "443":
zgrab_collection = rm_connector.get_zgrab_443_data_connection()
run_command = run_port_443_command
else:
zgrab_collection = rm_connector.get_zgrab_80_data_connection()
run_command = run_port_80_command
check_save_location("./json_p" + args.p)
global_zgrab_path = args.zgrab_path
threads = []
print("Creating " + str(args.t) + " threads")
for thread_id in range(1, args.t + 1):
thread = ZgrabThread(thread_id, global_work_queue, args.p, run_command,
zones_struct, zgrab_collection)
thread.start()
threads.append(thread)
thread_id += 1
print("Populating Queue")
global_queue_lock.acquire()
for ip in ips:
global_work_queue.put(ip)
global_queue_lock.release()
# Wait for queue to empty
while not global_work_queue.empty():
pass
# Notify threads it's time to exit
global_exit_flag = 1
# Wait for all threads to complete
for t in threads:
t.join()
print("Exiting Main Thread")
# Remove last week's old entries
lastweek = datetime.now() - timedelta(days=7)
zgrab_collection.remove({
'ip': {
"$ne": "<nil>"
},
'timestamp': {
"$lt": lastweek
}
})
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
Begin Main...
"""
# The sources for which to remove expired entries
# Infoblox is handled separately
# Sonar RDNS is hard code below in a separate section
# {"source_name": date_difference_in_months}
sources = [{"name": "sonar_dns", "diff": -2},
{"name": "sonar_dns_saved", "diff": -2},
{"name": "ssl", "diff": -2},
{"name": "ssl_saved", "diff": -2},
{"name": "virustotal", "diff": -2},
{"name": "virustotal_saved", "diff": -2},
{"name": "UltraDNS", "diff": -2},
{"name": "UltraDNS_saved", "diff": -2},
{"name": "skms", "diff": -2},
{"name": "skms_saved", "diff": -2},
{"name": "marinus", "diff": -2},
{"name": "marinus_saved", "diff": -2},
{"name": "mx", "diff": -2},
{"name": "mx_saved", "diff": -2},
{"name": "common_crawl", "diff": -4},
{"name": "common_crawl_saved", "diff": -4}]
now = datetime.now()
print ("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
dns_manager = DNSManager.DNSManager(mongo_connector)
GDNS = GoogleDNS.GoogleDNS()
zones = ZoneManager.get_distinct_zones(mongo_connector)
jobs_collection = mongo_connector.get_jobs_connection()
# Get the date for today minus two months
d_minus_2m = monthdelta(datetime.now(), -2)
print("Removing SRDNS as of: " + str(d_minus_2m))
# Remove the old records
srdns_collection = mongo_connector.get_sonar_reverse_dns_connection()
srdns_collection.remove({'updated': {"$lt": d_minus_2m}})
# Before completely removing old entries, make an attempt to see if they are still valid.
# Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source.
# Rather than throw away valid information, this will archive it.
for entry in sources:
removal_date = monthdelta(datetime.now(), entry['diff'])
source = entry['name']
print("Removing " + source + " as of: " + str(removal_date))
last_domain = ""
results = all_dns_collection.find({'sources': {"$size": 1}, 'sources.source': source, 'sources.updated': {"$lt": removal_date}})
for result in results:
if result['fqdn'] != last_domain:
last_domain = result['fqdn']
dns_result = GDNS.fetch_DNS_records(result['fqdn'], GDNS.DNS_TYPES[result['type']])
if dns_result != []:
for dns_entry in dns_result:
if is_tracked_zone(dns_entry['fqdn'], zones):
new_entry={}
new_entry['updated'] = datetime.now()
new_entry['zone'] = result['zone']
new_entry['fqdn'] = dns_entry['fqdn']
new_entry['created'] = result['created']
new_entry['value'] = dns_entry['value']
new_entry['type'] = dns_entry['type']
new_entry['status'] = 'confirmed'
if 'sonar_timestamp' in result:
new_entry['sonar_timestamp'] = result['sonar_timestamp']
if source.endswith("_saved"):
dns_manager.insert_record(new_entry, source)
else:
dns_manager.insert_record(new_entry, source + "_saved")
dns_manager.remove_all_by_source_and_date(source, entry['diff'])
# Record status
jobs_collection.update_one({'job_name': 'remove_expired_entries'},
{'$currentDate': {"updated": True},
"$set": {'status': 'COMPLETE'}})
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
Begin Main
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
graphs_collection = mongo_connector.get_graphs_connection()
graphs_data_collection = mongo_connector.get_graphs_data_connection()
graphs_links_collection = mongo_connector.get_graphs_links_connection()
graphs_docs_collection = mongo_connector.get_graphs_docs_connection()
ip_manager = IPManager.IPManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, 'create_graphs2')
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
for zone in zones:
groups = []
graph = nx.Graph()
add_to_list(zone, groups)
graph.add_node(zone,
data_type="tld",
type=0,
depends=[],
dependedOnBy=[],
docs="<h1>Parent</h1>")
find_all_dns_by_zone(graph, zone, groups, dns_manager, ip_manager)
find_srdns_by_zone(graph, zone, groups, mongo_connector, ip_manager)
data = json_graph.node_link_data(graph)
reformat_data(data, zone, groups)
new_data = {}
new_data['directed'] = data['directed']
new_data['graph'] = data['graph']
new_data['multigraph'] = data['multigraph']
new_data['errs'] = []
config = {}
config['title'] = zone + " Network Map"
config['graph'] = {}
config['graph']['linkDistance'] = 150
config['graph']['charge'] = -400
config['graph']['height'] = 800
config['graph']['numColors'] = len(groups)
config['graph']['labelPadding'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['labelMargin'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['ticksWithoutCollisions'] = 50
config['graph_type'] = "tracked_domain"
config['types'] = {}
regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
regx = re.compile(regex_str)
for tgroup in groups:
group = tgroup.replace(REPLACE_CHAR, ".")
data_type = "tpd"
if group in zones:
data_type = "tracked_domain"
elif re.match(regx, group):
data_type = "cidr"
config['types'][tgroup] = {
"short": group,
"long": "A group from the network: " + group,
"data_type": data_type
}
config['constraints'] = []
tmp = int(math.ceil(math.sqrt(len(groups)))) + 1
x = []
y = []
for i in range(1, tmp):
val = round((i * 1.0) / tmp, 2)
x.append(str(val))
y.append(str(val))
x_pos = 0
y_pos = 0
for group in groups:
config['constraints'].append({
"has": {
"type": group
},
"type": "position",
"x": x[x_pos],
"y": y[y_pos]
})
x_pos = x_pos + 1
if x_pos >= len(x):
x_pos = 0
y_pos = y_pos + 1
config['jsonUrl'] = "/api/v1.0/graphs/" + zone
new_data['config'] = config
new_data['created'] = datetime.now()
new_data['zone'] = zone
new_docs_data = {}
new_docs_data['docs'] = {}
new_docs_data['zone'] = zone
new_docs_data['created'] = datetime.now()
new_graph_data = {}
new_graph_data['data'] = {}
for i in range(0, len(data['nodes'])):
new_graph_data['data'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)] = data['nodes'][i]
new_docs_data['docs'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)] = data['nodes'][i]['docs']
del new_graph_data['data'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)]['docs']
new_graph_data['created'] = datetime.now()
new_graph_data['zone'] = zone
new_graph_data['directed'] = data['directed']
new_graph_data['multigraph'] = data['multigraph']
new_graph_data['errs'] = []
new_links_data = {}
new_links_data['links'] = data['links']
new_links_data['created'] = datetime.now()
new_links_data['zone'] = zone
new_links_data['directed'] = data['directed']
new_links_data['multigraph'] = data['multigraph']
new_links_data['errs'] = []
try:
graphs_collection.remove({'zone': zone})
graphs_collection.insert_one(new_data)
graphs_data_collection.remove({'zone': zone})
graphs_data_collection.insert_one(new_graph_data)
graphs_links_collection.remove({'zone': zone})
graphs_links_collection.insert_one(new_links_data)
graphs_docs_collection.remove({'zone': zone})
graphs_docs_collection.insert_one(new_docs_data)
except:
logger.error("ERROR: Can't insert: " + zone)
time.sleep(1)
# Remove last week's old entries
# In theory, shouldn't do anything but being complete
lastweek = datetime.now() - timedelta(days=7)
graphs_collection.remove({'created': {"$lt": lastweek}})
graphs_data_collection.remove({'created': {"$lt": lastweek}})
graphs_links_collection.remove({'created': {"$lt": lastweek}})
graphs_docs_collection.remove({'created': {"$lt": lastweek}})
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete")
def main():
"""
Begin Main()
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
mongo_ct = mongo_connector.get_certificate_transparency_connection()
cert_graphs_collection = mongo_connector.get_cert_graphs_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector, 'create_cert_graphs')
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
parser = argparse.ArgumentParser(description='Creates and stores certificate graphs in the database based on one or more sources.')
parser.add_argument('--check_censys', action='store_true', default=False, required=False, help='Whether to check the Censys collection in the database')
parser.add_argument('--check_443_scans', action='store_true', default=False, required=False, help='Whether to check the zgrab collection in the database')
parser.add_argument('--check_ct_scans', action='store_true', default=False, required=False, help='Whether to check the CT collection in the database')
parser.add_argument('--zgrab_version', default=2, type=int, choices=[1, 2], metavar="version", help='The version of ZGrab used to collect data')
args = parser.parse_args()
if args.check_censys is True:
censys_collection = mongo_connector.get_censys_connection()
if args.check_443_scans is True:
zgrab_collection = mongo_connector.get_zgrab_443_data_connection()
for zone in zones:
logger.info("Creating: " + zone)
graph = nx.DiGraph()
certs_list = {}
if args.check_ct_scans:
certs_list = get_current_ct_certificates(mongo_ct, zone)
if args.check_censys:
certs_list = add_censys_certificates(censys_collection, zone, certs_list)
if args.check_443_scans:
if args.zgrab_version == 1:
certs_list = add_terminal_zgrab_certificates(zgrab_collection, zone, certs_list)
certs_list = add_initial_zgrab_certificates(zgrab_collection, zone, certs_list)
else:
certs_list = add_terminal_zgrab2_certificates(zgrab_collection, zone, certs_list)
certs_list = add_initial_zgrab2_certificates(zgrab_collection, zone, certs_list)
graph = create_nodes(graph, mongo_connector, zone, certs_list)
data = json_graph.node_link_data(graph)
my_data = {}
my_data['links'] = data['links']
my_data['nodes'] = data['nodes']
my_data['zone'] = zone
my_data['created'] = datetime.now()
cert_graphs_collection.remove({'zone': zone})
cert_graphs_collection.insert(my_data)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
dns_types = {
"a": 1,
"ns": 2,
"cname": 5,
"soa": 6,
"ptr": 12,
"hinfo": 13,
"mx": 15,
"txt": 16,
"aaaa": 28,
"srv": 33,
"naptr": 35,
"ds": 43,
"rrsig": 46,
"dnskey": 48,
}
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector, "marinus_dns")
jobs_manager.record_job_start()
dns_manager = DNSManager.DNSManager(mongo_connector)
zones = ZoneManager.get_distinct_zones(mongo_connector)
google_dns = GoogleDNS.GoogleDNS()
for zone in zones:
time.sleep(1)
for dtype, dnum in dns_types.items():
result = google_dns.fetch_DNS_records(zone, dnum)
if result == []:
logger.debug("No records found for " + zone)
else:
new_record = result[0]
new_record["status"] = "confirmed"
new_record["zone"] = zone
new_record["created"] = datetime.now()
logger.debug("Found " + dtype + " for: " + zone)
dns_manager.insert_record(new_record, "marinus")
logger.info("Starting SOA Search")
soa_searches = find_sub_zones(all_dns_collection)
for entry in soa_searches:
time.sleep(1)
result = google_dns.fetch_DNS_records(zone, dns_types["soa"])
if result != []:
new_record = result[0]
new_record["status"] = "confirmed"
new_record["zone"] = get_fld_from_value(entry, "")
new_record["created"] = datetime.now()
logger.debug("Found SOA: " + entry)
if new_record["zone"] != "":
dns_manager.insert_record(new_record, "marinus")
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
if is_running(os.path.basename(__file__)):
logger.warning("Already running...")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
parser = argparse.ArgumentParser(
description="Parse Sonar files based on CIDRs.")
parser.add_argument(
"--sonar_file_type",
choices=["dns-any", "dns-a", "rdns"],
required=True,
help='Specify "dns-any", "dns-a", or "rdns"',
)
parser.add_argument(
"--database",
choices=["local", "remote"],
required=False,
default="local",
help="Whether to use the local or remote DB",
)
args = parser.parse_args()
r7 = Rapid7.Rapid7()
if args.database == "remote":
mongo_connection = RemoteMongoConnector.RemoteMongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connection,
"get_sonar_data_dns")
else:
mongo_connection = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connection)
ip_manager = IPManager.IPManager(mongo_connection)
zones = ZoneManager.get_distinct_zones(mongo_connection)
logger.info("Zone length: " + str(len(zones)))
save_directory = "./files/"
check_save_location(save_directory)
# A session is necessary for the multi-step log-in process
s = requests.Session()
if args.sonar_file_type == "rdns":
jobs_manager = JobsManager.JobsManager(mongo_connection,
"get_data_by_cidr_rdns")
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "rdns", jobs_manager)
if html_parser.rdns_url == "":
logger.error("Unknown Error")
jobs_manager.record_job_error()
exit(0)
unzipped_rdns = download_remote_files(logger, s,
html_parser.rdns_url,
save_directory, jobs_manager)
update_rdns(logger, unzipped_rdns, mongo_connection, dns_manager,
ip_manager, zones)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
logger.info("RDNS Complete")
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns":
jobs_manager = JobsManager.JobsManager(mongo_connection,
"get_data_by_cidr_dns")
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.any_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.any_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, dns_manager, ip_manager,
zones)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
elif args.sonar_file_type == "dns-a":
jobs_manager = JobsManager.JobsManager(mongo_connection,
"get_data_by_cidr_dns-a")
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.a_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.a_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, dns_manager, ip_manager,
zones)
if html_parser.aaaa_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.aaaa_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, dns_manager, ip_manager,
zones)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
logger.info("DNS Complete")
jobs_manager.record_job_complete()
else:
logger.error("Unrecognized sonar_file_type option. Exiting...")
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
azure_connector = AzureConnector.AzureConnector()
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zone_ingestor = ZoneIngestor.ZoneIngestor()
jobs_manager = JobsManager.JobsManager(mongo_connector, "fetch_azure_dns")
jobs_manager.record_job_start()
current_zones = ZoneManager.get_distinct_zones(mongo_connector)
resource_client = azure_connector.get_resources_client()
resources = []
# The resource list is not currently used.
for item in resource_client.resource_groups.list():
resources.append(item.name)
dns_client = azure_connector.get_dns_client()
zones = dns_client.zones.list()
# The type of records the Azure DNS will let you configure
record_types = {
"A": "arecords",
"AAAA": "aaaa_records",
"MX": "mx_records",
"NS": "ns_records",
"PTR": "ptr_records",
"SRV": "srv_records",
"TXT": "txt_records",
"CNAME": "cname_record",
"SOA": "soa_record",
}
for zone in zones:
logger.info("Zone: " + zone.name)
data = split_id(zone.id)
if zone.zone_type == ZoneType.public:
logger.info(zone.name + " is public:")
if zone.name not in current_zones:
logger.debug("Creating zone: " + zone.name)
zone_ingestor.add_zone(zone.name, "azure:" + data["resourceGroups"])
try:
logger.info("ResourceGroup: " + data["resourceGroups"])
records = dns_client.record_sets.list_all_by_dns_zone(
data["resourceGroups"], zone.name
)
for entry in records:
# The record_data id value ends in rtype/rvalue so you must guess the rtype
record_data = split_id(entry.id)
for rtype in record_types:
if rtype in record_data:
results = extract_record_set_value(logger, rtype, entry)
for result in results:
result["zone"] = zone.name
result["created"] = datetime.now()
result["status"] = "confirmed"
dns_manager.insert_record(
result, "azure:" + data["resourceGroups"]
)
except:
logger.warning("No records found")
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Make database connections
mongo_connector = MongoConnector.MongoConnector()
ct_collection = mongo_connector.get_certificate_transparency_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector, "facebook_certs")
jobs_manager.record_job_start()
file_path = "/mnt/workspace/ct_facebook/"
fb_connector = FacebookConnector.FacebookConnector()
access_token = fb_connector.get_facebook_access_token()
zones = ZoneManager.get_distinct_zones(mongo_connector)
x509_parser = X509Parser.X509Parser()
parser = argparse.ArgumentParser(
description="Download DNS and/or certificate information from crt.sh.")
parser.add_argument(
"--fetch_cert_records",
choices=["dbAndSave", "dbOnly"],
default="dbAndSave",
help=
"Indicates whether to download the raw files or just record in the database",
)
parser.add_argument(
"--cert_save_location",
required=False,
default=file_path,
help=
"Indicates where to save the certificates on disk when choosing dbAndSave",
)
args = parser.parse_args()
check_save_location(args.cert_save_location)
save_location = args.cert_save_location
if not save_location.endswith("/"):
save_location = save_location + "/"
for zone in zones:
time.sleep(15)
results = fetch_domain(logger, jobs_manager, fb_connector,
access_token, zone)
if results is None:
logger.warning("ERROR looking up: " + zone)
continue
logger.info(zone + ": " + str(len(results)))
for result in results:
if args.fetch_cert_records == "dbAndSave":
cert_f = open(
save_location + zone + "_" + result["id"] + ".pem", "w")
cert_f.write(result["certificate_pem"])
cert_f.close()
cert = x509_parser.parse_data(result["certificate_pem"],
"facebook")
cert["facebook_id"] = result["id"]
if (ct_collection.count_documents(
{"fingerprint_sha256": cert["fingerprint_sha256"]}) == 0):
mongo_connector.perform_insert(ct_collection, cert)
else:
if (ct_collection.count_documents({
"fingerprint_sha256":
cert["fingerprint_sha256"],
"facebook_id":
result["id"],
"zones":
zone,
}) == 0):
ct_collection.update_one(
{"fingerprint_sha256": cert["fingerprint_sha256"]},
{
"$set": {
"marinus_updated": datetime.now(),
"facebook_id": result["id"],
},
"$addToSet": {
"zones": zone
},
},
)
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, "extract_mx_domains")
google_dns = GoogleDNS.GoogleDNS()
jobs_manager.record_job_start()
dns_names = []
round_two = []
zones = ZoneManager.get_distinct_zones(mongo_connector)
# Collect the list of domains from the MX Records
extract_mx_names(dns_names, dns_manager)
input_list = []
# Some MX records point to the third-party domains.
# Therefore, we filter to only the root domains that belong to the tracked company.
logger.info("Pre-filter list: " + str(len(dns_names)))
for hostname in dns_names:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
# Pause to prevent DoS-ing of Google's HTTPS DNS Service
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr["fqdn"], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr["fqdn"]}
record["zone"] = temp_zone
record["created"] = datetime.now()
record["type"] = ip_addr["type"]
record["value"] = ip_addr["value"]
record["status"] = "unknown"
input_list.append(record)
if ip_addr["type"] == "cname" and is_tracked_zone(
ip_addr["value"], zones
):
add_to_round_two(ip_addr["value"], round_two)
else:
logger.warning("Failed IP Lookup for: " + hostname)
else:
logger.warning("Failed match on zone for: " + hostname)
dead_dns_collection = mongo_connector.get_dead_dns_connection()
# Some DNS records will be CNAME records pointing to other tracked domains.
# This is a single level recursion to lookup those domains.
logger.info("Round Two list: " + str(len(round_two)))
for hostname in round_two:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr["fqdn"], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr["fqdn"]}
record["zone"] = temp_zone
record["created"] = datetime.now()
record["type"] = ip_addr["type"]
record["value"] = ip_addr["value"]
record["status"] = "unknown"
input_list.append(record)
else:
logger.warning("Failed IP Lookup for: " + hostname)
original_record = dns_manager.find_one({"fqdn": hostname}, "mx")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
else:
logger.warning("Failed match on zone for: " + hostname)
# Record all the results.
dns_manager.remove_by_source("mx")
logger.info("List length: " + str(len(input_list)))
for final_result in input_list:
dns_manager.insert_record(final_result, "mx")
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
global global_exit_flag
global global_zgrab_path
logger = LoggingUtil.create_log(__name__)
parser = argparse.ArgumentParser(
description="Launch zgrab against domains using port 80 or 443.")
parser.add_argument("-p",
choices=["443", "80"],
metavar="port",
help="The web port: 80 or 443")
parser.add_argument("-t",
default=5,
type=int,
metavar="threadCount",
help="The number of threads")
parser.add_argument(
"--zgrab_path",
default=global_zgrab_path,
metavar="zgrabVersion",
help="The version of ZGrab to use",
)
args = parser.parse_args()
if args.p == None:
logger.error("A port value (80 or 443) must be provided.")
exit(1)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
now = datetime.now()
logger.warning(str(now) + ": I am already running! Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
rm_connector = RemoteMongoConnector.RemoteMongoConnector()
all_dns_collection = rm_connector.get_all_dns_connection()
jobs_manager = JobsManager.JobsManager(rm_connector,
"zgrab_http_domain-" + args.p)
jobs_manager.record_job_start()
if args.p == "443":
zgrab_collection = rm_connector.get_zgrab_443_data_connection()
run_command = run_port_443_command
else:
zgrab_collection = rm_connector.get_zgrab_80_data_connection()
run_command = run_port_80_command
check_save_location("./json_p" + args.p)
global_zgrab_path = args.zgrab_path
zones = ZoneManager.get_distinct_zones(rm_connector)
ip_manager = IPManager.IPManager(rm_connector)
for zone in zones:
global_exit_flag = 0
domains = get_domains(all_dns_collection, ip_manager, zone)
if len(domains) == 0:
continue
num_threads = args.t
if len(domains) < args.t:
num_threads = len(domains)
logger.debug("Creating " + str(num_threads) + " threads")
threads = []
for thread_id in range(1, num_threads + 1):
thread = ZgrabThread(
thread_id,
global_work_queue,
args.p,
run_command,
zone,
zgrab_collection,
)
thread.start()
threads.append(thread)
thread_id += 1
logger.debug(zone + " length: " + str(len(domains)))
logger.info("Populating Queue")
global_queue_lock.acquire()
for domain in domains:
global_work_queue.put(domain)
global_queue_lock.release()
# Wait for queue to empty
while not global_work_queue.empty():
pass
logger.info("Queue empty")
# Notify threads it's time to exit
global_exit_flag = 1
# Wait for all threads to complete
for t in threads:
t.join()
# Remove last week's old entries
lastweek = datetime.now() - timedelta(days=7)
zgrab_collection.delete_many({
"domain": {
"$ne": "<nil>"
},
"timestamp": {
"$lt": lastweek
}
})
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
if is_running(os.path.basename(__file__)):
print("Already running...")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
r7 = Rapid7.Rapid7()
zones = ZoneManager.get_distinct_zones(mongo_connector)
parser = argparse.ArgumentParser(
description='Parse Sonar files based on domain zones.')
parser.add_argument('--sonar_file_type',
required=True,
help='Specify "dns-any", "dns-a", or "rdns"')
args = parser.parse_args()
jobs_collection = mongo_connector.get_jobs_connection()
# A session is necessary for the multi-step log-in process
s = requests.Session()
if args.sonar_file_type == "rdns":
now = datetime.now()
print("Updating RDNS: " + str(now))
jobs_collection.update_one({'job_name': 'get_sonar_data_rdns'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'RUNNING'
}
})
try:
html_parser = r7.find_file_locations(s, "rdns",
"get_sonar_data_rdns",
jobs_collection)
if html_parser.rdns_url == "":
now = datetime.now()
print("Unknown Error: " + str(now))
jobs_collection.update_one({'job_name': 'get_sonar_data_rdns'},
{
'$currentDate': {
"updated": True
},
"$set": {
'status': 'ERROR'
}
})
exit(0)
unzipped_rdns = download_remote_files(s, html_parser.rdns_url,
global_data_dir,
"get_sonar_data_rdns",
jobs_collection)
update_rdns(unzipped_rdns, zones, mongo_connector)
except Exception as ex:
now = datetime.now()
print("Unknown error occured at: " + str(now))
print("Unexpected error: " + str(ex))
jobs_collection.update_one({'job_name': 'get_sonar_data_rdns'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'ERROR'
}
})
exit(0)
# Update used instead of update_one due to old pymongo library on IT machines
jobs_collection.update_one({'job_name': 'get_sonar_data_rdns'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
elif args.sonar_file_type == "dns-any":
now = datetime.now()
print("Updating DNS: " + str(now))
jobs_collection.update_one({'job_name': 'get_sonar_data_dns-any'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'RUNNING'
}
})
try:
html_parser = r7.find_file_locations(s, "fdns",
"get_sonar_data_dns-any",
jobs_collection)
if html_parser.any_url != "":
unzipped_dns = download_remote_files(s, html_parser.any_url,
global_data_dir,
"get_sonar_data_dns-any",
jobs_collection)
update_dns(unzipped_dns, zones, global_dns_manager)
except Exception as ex:
now = datetime.now()
print("Unknown error occured at: " + str(now))
print("Unexpected error: " + str(ex))
jobs_collection.update_one({'job_name': 'get_sonar_data_dns-any'},
{
'$currentDate': {
"updated": True
},
"$set": {
'status': 'ERROR'
}
})
exit(0)
# Update used instead of update_one due to old pymongo library on IT machines
jobs_collection.update_one({'job_name': 'get_sonar_data_dns-any'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
elif args.sonar_file_type == "dns-a":
now = datetime.now()
print("Updating DNS: " + str(now))
jobs_collection.update_one({'job_name': 'get_sonar_data_dns-a'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'RUNNING'
}
})
try:
html_parser = r7.find_file_locations(s, "fdns",
"get_sonar_data_dns-a",
jobs_collection)
if html_parser.a_url != "":
unzipped_dns = download_remote_files(s, html_parser.a_url,
global_data_dir,
"get_sonar_data_dns-a",
jobs_collection)
update_dns(unzipped_dns, zones, global_dns_manager)
if html_parser.aaaa_url != "":
unzipped_dns = download_remote_files(s, html_parser.aaaa_url,
global_data_dir,
"get_sonar_data_dns-a",
jobs_collection)
update_dns(unzipped_dns, zones, global_dns_manager)
except Exception as ex:
now = datetime.now()
print("Unknown error occured at: " + str(now))
print("Unexpected error: " + str(ex))
jobs_collection.update_one({'job_name': 'get_sonar_data_dns-a'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'ERROR'
}
})
exit(0)
# Update used instead of update_one due to old pymongo library on IT machines
jobs_collection.update_one({'job_name': 'get_sonar_data_dns-a'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
else:
print("Unrecognized sonar_file_type option. Exiting...")
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
The main thread for this program.
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
jobs_collection = mongo_connector.get_jobs_connection()
zones = ZoneManager.get_distinct_zones(mongo_connector)
tpds = get_tpds(mongo_connector)
# For third-party-domain in the list of third-party-domains
for tpd in tpds:
groups = []
graph = nx.DiGraph()
add_to_list(tpd, groups)
# A space is added because sometimes the tpd is the same as the end target node
graph.add_node(tpd + " ",
data_type="tld",
type=0,
depends=[],
dependedOnBy=[],
docs="<h1>Parent</h1>")
# Get the zones associated with the tpd
find_zones_by_tld(graph, tpd, groups, mongo_connector)
data = json_graph.node_link_data(graph)
reformat_data(data, tpd, groups)
new_data = {}
new_data['directed'] = data['directed']
new_data['graph'] = data['graph']
new_data['multigraph'] = data['multigraph']
new_data['errs'] = []
new_data['links'] = data['links']
new_data['data'] = {}
for i in range(0, len(data['nodes'])):
new_data['data'][data['nodes'][i]['id'].replace(
".", REPLACE_CHAR)] = data['nodes'][i]
for entry in new_data['data']:
for dep in new_data['data'][entry]['depends']:
if new_data['data'][entry]['name'] not in new_data['data'][
dep.replace(".", REPLACE_CHAR)]['dependedOnBy']:
new_data['data'][dep.replace(
".", REPLACE_CHAR)]['dependedOnBy'].append(
new_data['data'][entry]['name'])
for dep in new_data['data'][entry]['dependedOnBy']:
if new_data['data'][entry]['name'] not in new_data['data'][
dep.replace(".", REPLACE_CHAR)]['depends']:
new_data['data'][dep.replace(
".", REPLACE_CHAR)]['depends'].append(
new_data['data'][entry]['name'])
for entry in new_data['data']:
new_data['data'][entry]['docs'] = build_docs(
new_data['data'][entry], tpd, groups)
config = {}
config['title'] = tpd + " Network Map"
config['graph'] = {}
config['graph']['linkDistance'] = 150
config['graph']['charge'] = -400
config['graph']['height'] = 800
config['graph']['numColors'] = len(groups)
config['graph']['labelPadding'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['labelMargin'] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config['graph']['ticksWithoutCollisions'] = 50
config['graph_type'] = "tpd"
config['types'] = {}
regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
regx = re.compile(regex_str)
for tgroup in groups:
data_type = "tpd"
group = tgroup.replace(REPLACE_CHAR, ".")
if group in zones:
data_type = "tracked_domain"
elif re.match(regx, group):
data_type = "cidr"
config['types'][tgroup] = {
"short": group,
"long": "A group from the network: " + group,
"data_type": data_type
}
config['constraints'] = []
tmp = int(math.ceil(math.sqrt(len(groups)))) + 1
x = []
y = []
for i in range(1, tmp):
val = round((i * 1.0) / tmp, 2)
x.append(str(val))
y.append(str(val))
x_pos = 0
y_pos = 0
for group in groups:
config['constraints'].append({
"has": {
"type": group
},
"type": "position",
"x": x[x_pos],
"y": y[y_pos]
})
x_pos = x_pos + 1
if x_pos >= len(x):
x_pos = 0
y_pos = y_pos + 1
config['jsonUrl'] = "/api/v1.0/tpd_graphs/" + tpd
new_data['config'] = config
new_data['created'] = datetime.now()
new_data['zone'] = tpd
tpd_graphs_collection = mongo_connector.get_tpd_graphs_connection()
tpd_graphs_collection.remove({'zone': tpd})
try:
tpd_graphs_collection.insert_one(new_data)
except:
print("ERROR: Could not insert " + tpd)
time.sleep(1)
# Remove last week's old entries
lastweek = datetime.now() - timedelta(days=7)
tpd_graphs_collection.remove({'created': {"$lt": lastweek}})
# Record status
jobs_collection.update_one({'job_name': 'create_tpd_graphs'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Complete: " + str(now))
def main():
"""
The main thread for this program.
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
jobs_manager = JobsManager.JobsManager(mongo_connector,
"create_tpd_graphs")
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
tpds = get_tpds(mongo_connector)
# For third-party-domain in the list of third-party-domains
for tpd in tpds:
groups = []
graph = nx.DiGraph()
add_to_list(tpd, groups)
# A space is added because sometimes the tpd is the same as the end target node
graph.add_node(
tpd + " ",
data_type="tld",
type=0,
depends=[],
dependedOnBy=[],
docs="<h1>Parent</h1>",
)
# Get the zones associated with the tpd
find_zones_by_tld(graph, tpd, groups, mongo_connector)
data = json_graph.node_link_data(graph)
reformat_data(data, tpd, groups)
new_data = {}
new_data["directed"] = data["directed"]
new_data["graph"] = data["graph"]
new_data["multigraph"] = data["multigraph"]
new_data["errs"] = []
new_data["links"] = data["links"]
new_data["data"] = {}
for i in range(0, len(data["nodes"])):
new_data["data"][data["nodes"][i]["id"].replace(
".", REPLACE_CHAR)] = data["nodes"][i]
for entry in new_data["data"]:
for dep in new_data["data"][entry]["depends"]:
if (new_data["data"][entry]["name"]
not in new_data["data"][dep.replace(
".", REPLACE_CHAR)]["dependedOnBy"]):
new_data["data"][dep.replace(
".", REPLACE_CHAR)]["dependedOnBy"].append(
new_data["data"][entry]["name"])
for dep in new_data["data"][entry]["dependedOnBy"]:
if (new_data["data"][entry]["name"]
not in new_data["data"][dep.replace(
".", REPLACE_CHAR)]["depends"]):
new_data["data"][dep.replace(
".", REPLACE_CHAR)]["depends"].append(
new_data["data"][entry]["name"])
for entry in new_data["data"]:
new_data["data"][entry]["docs"] = build_docs(
new_data["data"][entry], tpd, groups)
config = {}
config["title"] = tpd + " Network Map"
config["graph"] = {}
config["graph"]["linkDistance"] = 150
config["graph"]["charge"] = -400
config["graph"]["height"] = 800
config["graph"]["numColors"] = len(groups)
config["graph"]["labelPadding"] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config["graph"]["labelMargin"] = {
"left": 3,
"right": 3,
"top": 2,
"bottom": 2
}
config["graph"]["ticksWithoutCollisions"] = 50
config["graph_type"] = "tpd"
config["types"] = {}
regex_str = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
regx = re.compile(regex_str)
for tgroup in groups:
data_type = "tpd"
group = tgroup.replace(REPLACE_CHAR, ".")
if group in zones:
data_type = "tracked_domain"
elif re.match(regx, group):
data_type = "cidr"
config["types"][tgroup] = {
"short": group,
"long": "A group from the network: " + group,
"data_type": data_type,
}
config["constraints"] = []
tmp = int(math.ceil(math.sqrt(len(groups)))) + 1
x = []
y = []
for i in range(1, tmp):
val = round((i * 1.0) / tmp, 2)
x.append(str(val))
y.append(str(val))
x_pos = 0
y_pos = 0
for group in groups:
config["constraints"].append({
"has": {
"type": group
},
"type": "position",
"x": x[x_pos],
"y": y[y_pos],
})
x_pos = x_pos + 1
if x_pos >= len(x):
x_pos = 0
y_pos = y_pos + 1
config["jsonUrl"] = "/api/v1.0/tpd_graphs/" + tpd
new_data["config"] = config
new_data["created"] = datetime.now()
new_data["zone"] = tpd
tpd_graphs_collection = mongo_connector.get_tpd_graphs_connection()
tpd_graphs_collection.delete_one({"zone": tpd})
try:
tpd_graphs_collection.insert_one(new_data)
except:
logger.error("ERROR: Could not insert " + tpd)
time.sleep(1)
# Remove last week's old entries
lastweek = datetime.now() - timedelta(days=7)
tpd_graphs_collection.delete_many({"created": {"$lt": lastweek}})
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
all_dns_collection = mongo_connector.get_all_dns_connection()
dns_manager = DNSManager.DNSManager(mongo_connector)
GDNS = GoogleDNS.GoogleDNS()
ip_manager = IPManager.IPManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
'remove_expired_entries')
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
# The sources for which to remove expired entries
results = mongo_connector.perform_distinct(all_dns_collection,
'sources.source')
sources = []
for source in results:
temp = {}
temp['name'] = source
if "common_crawl" in source:
temp['diff'] = -4
else:
temp['diff'] = -2
sources.append(temp)
# Before completely removing old entries, make an attempt to see if they are still valid.
# Occasionally, a host name will still be valid but, for whatever reason, is no longer tracked by a source.
# Rather than throw away valid information, this will archive it.
for entry in sources:
removal_date = monthdelta(datetime.now(), entry['diff'])
source = entry['name']
logger.debug("Removing " + source + " as of: " + str(removal_date))
last_domain = ""
results = all_dns_collection.find({
'sources': {
"$size": 1
},
'sources.source': source,
'sources.updated': {
"$lt": removal_date
}
})
for result in results:
if result['fqdn'] != last_domain:
last_domain = result['fqdn']
lookup_int = get_lookup_int(logger, result, GDNS)
dns_result = GDNS.fetch_DNS_records(result['fqdn'], lookup_int)
if dns_result != []:
insert_current_results(dns_result, dns_manager, zones,
result, source)
dns_manager.remove_all_by_source_and_date(source, entry['diff'])
# Get the date for today minus two months
d_minus_2m = monthdelta(datetime.now(), -2)
logger.info("Removing SRDNS as of: " + str(d_minus_2m))
# Remove the old records
srdns_collection = mongo_connector.get_sonar_reverse_dns_connection()
srdns_collection.remove({'updated': {"$lt": d_minus_2m}})
ip_manager.delete_records_by_date(d_minus_2m)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
if is_running(os.path.basename(__file__)):
logger.warning("Already running...")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
r7 = Rapid7.Rapid7()
zones = ZoneManager.get_distinct_zones(mongo_connector)
save_directory = "./files/"
parser = argparse.ArgumentParser(
description='Parse Sonar files based on domain zones.')
parser.add_argument('--sonar_file_type',
required=True,
help='Specify "dns-any", "dns-a", or "rdns"')
args = parser.parse_args()
check_save_location(save_directory)
# A session is necessary for the multi-step log-in process
s = requests.Session()
if args.sonar_file_type == "rdns":
logger.info("Updating RDNS")
jobs_manager = JobsManager.JobsManager(mongo_connector,
'get_sonar_data_rdns')
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "rdns", jobs_manager)
if html_parser.rdns_url == "":
logger.error("Unknown Error")
jobs_manager.record_job_error()
exit(0)
unzipped_rdns = download_remote_files(logger, s,
html_parser.rdns_url,
save_directory, jobs_manager)
update_rdns(logger, unzipped_rdns, zones, mongo_connector)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns-any":
logger.info("Updating DNS")
jobs_manager = JobsManager.JobsManager(mongo_connector,
'get_sonar_data_dns-any')
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.any_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.any_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, global_dns_manager)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns-a":
logger.info("Updating DNS")
jobs_manager = JobsManager.JobsManager(mongo_connector,
'get_sonar_data_dns-a')
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.a_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.a_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, global_dns_manager)
if html_parser.aaaa_url != "":
unzipped_dns = download_remote_files(logger, s,
html_parser.aaaa_url,
save_directory,
jobs_manager)
update_dns(logger, unzipped_dns, zones, global_dns_manager)
except Exception as ex:
logger.error("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
jobs_manager.record_job_complete()
else:
logger.error("Unrecognized sonar_file_type option. Exiting...")
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Connect to the remote databases
mongo_connector = MongoConnector.MongoConnector()
rm_connector = RemoteMongoConnector.RemoteMongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zones = ZoneManager.get_distinct_zones(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, "remote_download")
jobs_manager.record_job_start()
remote_jobs_collection = rm_connector.get_jobs_connection()
# Check the status of the Censys job on the remote database
try:
status = remote_jobs_collection.find_one({"job_name": "censys"})
except:
logger.error("Can not connect to remote database")
jobs_manager.record_job_error()
exit(1)
if (status is not None and "status" in status
and status["status"] != jobs_manager.COMPLETE):
logger.info("Censys scans status is not COMPLETE")
elif (status is not None and "status" in status
and status["status"] == jobs_manager.COMPLETE):
# Get connections to the relevant collections.
censys_collection = mongo_connector.get_zgrab_443_data_connection()
remote_censys_collection = rm_connector.get_zgrab_443_data_connection()
download_censys_scan_info(logger, censys_collection,
remote_censys_collection)
# Tell the remote database that is safe to start processing the next Censys file
remote_jobs_collection.update_one(
{"job_name": "censys"},
{
"$currentDate": {
"updated": True
},
"$set": {
"status": jobs_manager.READY
}
},
)
# Get connections to the relevant HTTPS collections.
zgrab_443_data_collection = mongo_connector.get_zgrab_443_data_connection()
remote_zgrab_443_data_collection = rm_connector.get_zgrab_443_data_connection(
)
download_zgrab_info(logger, zgrab_443_data_collection,
remote_zgrab_443_data_collection)
# Get connections to the relevant HTTP collections.
zgrab_80_data_collection = mongo_connector.get_zgrab_80_data_connection()
remote_zgrab_80_data_collection = rm_connector.get_zgrab_80_data_connection(
)
download_zgrab_info(logger, zgrab_80_data_collection,
remote_zgrab_80_data_collection)
# Get connections to the relevant port collections.
zgrab_port_data_collection = mongo_connector.get_zgrab_port_data_connection(
)
remote_zgrab_port_data_collection = rm_connector.get_zgrab_port_data_connection(
)
download_zgrab_port_info(logger, zgrab_port_data_collection,
remote_zgrab_port_data_collection)
# Download latest whois information
status = remote_jobs_collection.find_one({"job_name": "whois_lookups"})
if status["status"] == jobs_manager.COMPLETE:
whois_collection = mongo_connector.get_whois_connection()
remote_whois_collection = rm_connector.get_whois_connection()
download_whois_data(logger, whois_collection, remote_whois_collection)
remote_jobs_collection.update_one(
{"job_name": "whois"}, {"$set": {
"status": jobs_manager.READY
}})
# Download Amass results
amass_collection = mongo_connector.get_owasp_amass_connection()
remote_amass_collection = rm_connector.get_owasp_amass_connection()
download_amass_data(logger, amass_collection, remote_amass_collection,
dns_manager, zones)
# Download the status of the remote jobs
download_jobs_status(logger, jobs_manager._jobs_collection,
remote_jobs_collection)
# Download remote sonar DNS findings
download_sonar_dns(logger, dns_manager, rm_connector)
# Download remote sonar RDNS findings
download_sonar_rdns(logger, mongo_connector, rm_connector)
# Update the local jobs database to done
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Set up the common objects
mongo_connector = MongoConnector.MongoConnector()
ct_collection = mongo_connector.get_certificate_transparency_connection()
zones = ZoneManager.get_distinct_zones(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, "get_crt_sh")
jobs_manager.record_job_start()
save_location = "/mnt/workspace/crt_sh"
download_method = 'dbAndSave'
parser = argparse.ArgumentParser(
description='Download DNS and/or certificate information from crt.sh.')
parser.add_argument(
'--fetch_dns_records',
action='store_true',
help='Indicates whether to add DNS entries to the database')
parser.add_argument(
'--download_methods',
choices=['dbAndSave', 'dbOnly'],
default=download_method,
help=
'Indicates whether to download the raw files or just record in the database.'
)
parser.add_argument(
'--cert_save_location',
required=False,
default=save_location,
help=
'Indicates where to save the certificates on disk when choosing dbAndSave'
)
args = parser.parse_args()
if args.cert_save_location:
save_location = args.cert_save_location
if not save_location.endswith("/"):
save_location = save_location + "/"
if args.download_methods == 'dbAndSave':
check_save_location(save_location)
for zone in zones:
# Pace out requests so as not to DoS crt.sh and Google DNS
time.sleep(5)
# This could be done with backoff but we don't want to be overly aggressive.
json_result = make_https_request(
logger, "https://crt.sh/?q=%25." + zone + "&output=json")
if json_result is None:
logger.warning("Can't find result for: " + zone)
json_result = "{}"
json_data = json.loads(json_result)
new_names = []
new_ids = []
for entry in json_data:
if entry['min_cert_id'] not in new_ids:
new_ids.append(entry['min_cert_id'])
if "*" not in entry["name_value"] and entry[
"name_value"] not in new_names:
new_names.append(entry["name_value"])
if args.fetch_dns_records:
add_new_domain_names(new_names, zones, mongo_connector)
if args.download_methods == "dbAndSave":
add_new_certificate_values(logger, new_ids, ct_collection, zones,
save_location)
elif args.download_methods == "dbOnly":
add_new_certificate_values(logger, new_ids, ct_collection, zones,
None)
# Set isExpired for any entries that have recently expired.
ct_collection.update(
{
"not_after": {
"$lt": datetime.utcnow()
},
"isExpired": False
}, {"$set": {
"isExpired": True
}},
multi=True)
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
google_dns = GoogleDNS.GoogleDNS()
jobs_collection = mongo_connector.get_jobs_connection()
dns_names = []
round_two = []
zones = ZoneManager.get_distinct_zones(mongo_connector)
# Collect the list of domains from the MX Records
extract_mx_names(dns_names, dns_manager)
input_list = []
# Some MX records point to the third-party domains.
# Therefore, we filter to only the root domains that belong to the tracked company.
print("Pre-filter list: " + str(len(dns_names)))
for hostname in dns_names:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
# Pause to prevent DoS-ing of Google's HTTPS DNS Service
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
if ip_addr['type'] == "cname" and is_tracked_zone(
ip_addr['value'], zones):
add_to_round_two(ip_addr['value'], round_two)
else:
print("Failed IP Lookup for: " + hostname)
else:
print("Failed match on zone for: " + hostname)
dead_dns_collection = mongo_connector.get_dead_dns_connection()
# Some DNS records will be CNAME records pointing to other tracked domains.
# This is a single level recursion to lookup those domains.
print("Round Two list: " + str(len(round_two)))
for hostname in round_two:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr['fqdn'], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = temp_zone
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unknown'
input_list.append(record)
else:
print("Failed IP Lookup for: " + hostname)
original_record = dns_manager.find_one({"fqdn": hostname},
"mx")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
else:
print("Failed match on zone for: " + hostname)
# Record all the results.
dns_manager.remove_by_source("mx")
print("List length: " + str(len(input_list)))
for final_result in input_list:
dns_manager.insert_record(final_result, "mx")
# Record status
jobs_collection.update_one({'job_name': 'extract_mx_domains'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Ending: " + str(now))
def main():
now = datetime.now()
print("Starting: " + str(now))
azure_connector = AzureConnector.AzureConnector()
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
zone_ingestor = ZoneIngestor.ZoneIngestor()
current_zones = ZoneManager.get_distinct_zones(mongo_connector)
resource_client = azure_connector.get_resources_client()
resources = []
# The resource list is not currently used.
for item in resource_client.resource_groups.list():
resources.append(item.name)
dns_client = azure_connector.get_dns_client()
zones = dns_client.zones.list()
# The type of records the Azure DNS will let you configure
record_types = {
'A': 'arecords',
'AAAA': 'aaaa_records',
'MX': 'mx_records',
'NS': 'ns_records',
'PTR': 'ptr_records',
'SRV': 'srv_records',
'TXT': 'txt_records',
'CNAME': 'cname_record',
'SOA': 'soa_record'
}
for zone in zones:
print("Zone: " + zone.name)
data = split_id(zone.id)
if zone.zone_type == ZoneType.public:
print(zone.name + " is public:")
if zone.name not in current_zones:
print("Creating zone: " + zone.name)
zone_ingestor.add_zone(zone.name,
"azure:" + data["resourceGroups"])
try:
print("ResourceGroup: " + data["resourceGroups"])
records = dns_client.record_sets.list_all_by_dns_zone(
data["resourceGroups"], zone.name)
for entry in records:
# The record_data id value ends in rtype/rvalue so you must guess the rtype
record_data = split_id(entry.id)
for rtype in record_types:
if rtype in record_data:
results = extract_record_set_value(rtype, entry)
for result in results:
result['zone'] = zone.name
result['created'] = datetime.now()
result['status'] = 'confirmed'
dns_manager.insert_record(
result, "azure:" + data["resourceGroups"])
except:
print("No records found")
def main():
"""
Begin main...
"""
logger = LoggingUtil.create_log(__name__)
mongo_connector = MongoConnector.MongoConnector()
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
jobs_manager = JobsManager.JobsManager(mongo_connector, 'owasp_amass')
zones = ZoneManager.get_distinct_zones(mongo_connector)
dns_manager = DNSManager.DNSManager(mongo_connector)
output_dir = "./amass_files/"
arg_parser = argparse.ArgumentParser(
description=
'Run the OWASP Amass tool and store the results in the database.')
arg_parser.add_argument(
'--config_file',
required=False,
help='An optional Amass config file. Otherwise, defaults will be used.'
)
arg_parser.add_argument('--amass_path',
required=True,
help='The path to the amass binary')
arg_parser.add_argument('--output_dir',
default=output_dir,
help="The path where to save Amass files.")
arg_parser.add_argument('--amass_version',
type=int,
default=3,
help='The version of OWASP Amass being used.')
arg_parser.add_argument(
'--sleep',
type=int,
default=5,
help=
'Sleep time in seconds between amass runs so as not to overuse service limits.'
)
args = arg_parser.parse_args()
if not os.path.isfile(args.amass_path):
logger.error("Incorrect amass_path argument provided")
exit(1)
if 'config_file' in args and not os.path.isfile(args.config_file):
logger.error("Incorrect config_file location")
exit(1)
if 'output_dir' in args:
output_dir = args.output_dir
if not output_dir.endswith("/"):
output_dir = output_dir + "/"
check_save_location(output_dir)
jobs_manager.record_job_start()
# If the job died half way through, you can skip over domains that were already processed
# when you restart the script.
new_zones = []
for zone in zones:
if not os.path.isfile(output_dir + zone + "-do.json"):
new_zones.append(zone)
for zone in new_zones:
# Pace out calls to the Amass services
time.sleep(args.sleep)
command_line = []
command_line.append(args.amass_path)
if int(args.amass_version) >= 3:
command_line.append("enum")
if args.config_file:
command_line.append("-config")
command_line.append(args.config_file)
command_line.append("-d")
command_line.append(zone)
command_line.append("-src")
command_line.append("-ip")
command_line.append("-o")
command_line.append(output_dir + zone + "-do.json")
try:
subprocess.check_call(command_line)
except subprocess.CalledProcessError as e:
# Even when there is an error, there will likely still be results.
# We can continue with the data that was collected thus far.
logger.warning("ERROR: Amass run exited with a non-zero status: " +
str(e))
if os.path.isfile(output_dir + zone + "-do.json"):
output = open(output_dir + zone + "-do.json", "r")
json_data = []
for line in output:
try:
json_data.append(json.loads(line))
except:
logger.warning("Amass wrote an incomplete line: " +
str(line))
output.close()
for finding in json_data:
if 'type' in finding and finding[
'type'] == 'infrastructure' or finding[
'type'] == 'domain':
# Not currently recording
continue
elif is_tracked_zone(finding['domain'], zones):
record_finding(dns_manager, finding)
else:
# logger.debug("Skipping: " + finding['domain'] + " type: " + finding['type'])
pass
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector,
"extract_vt_domains")
google_dns = GoogleDNS.GoogleDNS()
jobs_manager.record_job_start()
round_two = []
zones = ZoneManager.get_distinct_zones(mongo_connector)
vt_collection = mongo_connector.get_virustotal_connection()
vt_results = vt_collection.find({
"subdomains": {
"$exists": True
}
}, {
"zone": 1,
"subdomains": 1
}).batch_size(30)
input_list = []
# For each result found in the first pass across VirusTotal
for result in vt_results:
# Pause to prevent DoS-ing of Google's HTTPS DNS Service
time.sleep(1)
for hostname in result["subdomains"]:
ips = google_dns.fetch_DNS_records(hostname)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr["fqdn"], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr["fqdn"]}
record["zone"] = temp_zone
record["created"] = datetime.now()
record["type"] = ip_addr["type"]
record["value"] = ip_addr["value"]
record["status"] = "unknown"
input_list.append(record)
if ip_addr["type"] == "cname" and is_tracked_zone(
ip_addr["value"], zones):
add_to_list(ip_addr["value"], round_two)
else:
logger.warning("Failed IP Lookup for: " + hostname)
dead_dns_collection = mongo_connector.get_dead_dns_connection()
# For each tracked CName result found in the first pass across VirusTotal
logger.info("Round Two length: " + str(len(round_two)))
for hostname in round_two:
zone = get_tracked_zone(hostname, zones)
if zone != None:
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
temp_zone = get_tracked_zone(ip_addr["fqdn"], zones)
if temp_zone is not None:
record = {"fqdn": ip_addr["fqdn"]}
record["zone"] = temp_zone
record["created"] = datetime.now()
record["type"] = ip_addr["type"]
record["value"] = ip_addr["value"]
record["status"] = "unknown"
input_list.append(record)
else:
original_record = dns_manager.find_one({"fqdn": hostname},
"virustotal")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
logger.warning("Failed IP Lookup for: " + hostname)
else:
logger.warning("Failed match on zone for: " + hostname)
# Update the database
dns_manager.remove_by_source("virustotal")
logger.info("List length: " + str(len(input_list)))
for final_result in input_list:
dns_manager.insert_record(final_result, "virustotal")
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
if is_running(os.path.basename(__file__)):
print("Already running...")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
r7 = Rapid7.Rapid7()
mongo_connection = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connection)
ip_manager = IPManager.IPManager(mongo_connection)
rdns_collection = mongo_connection.get_sonar_reverse_dns_connection()
zones = ZoneManager.get_distinct_zones(mongo_connection)
print ("Zone length: " + str(len(zones)))
save_directory = "./files/"
parser = argparse.ArgumentParser(description='Parse Sonar files based on CIDRs.')
parser.add_argument('--sonar_file_type', required=True, help='Specify "dns" or "rdns"')
args = parser.parse_args()
check_save_location(save_directory)
# A session is necessary for the multi-step log-in process
s = requests.Session()
if args.sonar_file_type == "rdns":
jobs_manager = JobsManager.JobsManager(mongo_connection, 'get_data_by_cidr_rdns')
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "rdns", jobs_manager)
if html_parser.rdns_url == "":
now = datetime.now()
print ("Unknown Error: " + str(now))
jobs_manager.record_job_error()
exit(0)
unzipped_rdns = download_remote_files(s, html_parser.rdns_url, save_directory, jobs_manager)
update_rdns(unzipped_rdns, rdns_collection, dns_manager, ip_manager, zones)
except Exception as ex:
now = datetime.now()
print ("Unknown error occured at: " + str(now))
print ("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
now = datetime.now()
print ("RDNS Complete: " + str(now))
jobs_manager.record_job_complete()
elif args.sonar_file_type == "dns":
jobs_manager = JobsManager.JobsManager(mongo_connection, 'get_data_by_cidr_dns')
jobs_manager.record_job_start()
try:
html_parser = r7.find_file_locations(s, "fdns", jobs_manager)
if html_parser.any_url != "":
unzipped_dns = download_remote_files(s, html_parser.any_url, save_directory, jobs_manager)
update_dns(unzipped_dns, dns_manager, ip_manager, zones)
if html_parser.a_url != "":
unzipped_dns = download_remote_files(s, html_parser.a_url, save_directory, jobs_manager)
update_dns(unzipped_dns, dns_manager, ip_manager, zones)
if html_parser.aaaa_url != "":
unzipped_dns = download_remote_files(s, html_parser.aaaa_url, save_directory, jobs_manager)
update_dns(unzipped_dns, dns_manager, ip_manager, zones)
except Exception as ex:
now = datetime.now()
print ("Unknown error occured at: " + str(now))
print ("Unexpected error: " + str(ex))
jobs_manager.record_job_error()
exit(0)
now = datetime.now()
print ("DNS Complete: " + str(now))
jobs_manager.record_job_complete()
else:
print ("Unrecognized sonar_file_type option. Exiting...")
now = datetime.now()
print ("Complete: " + str(now))
def main():
"""
Begin Main...
"""
logger = LoggingUtil.create_log(__name__)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Create an instance of the VirusTotal class
vt_instance = VirusTotal.VirusTotal()
# Get collections for the queries
mongo_connector = MongoConnector.MongoConnector()
vt_collection = mongo_connector.get_virustotal_connection()
jobs_manager = JobsManager.JobsManager(mongo_connector,
"get_virustotal_data")
jobs_manager.record_job_start()
# Collect the list of tracked TLDs
zones = ZoneManager.get_distinct_zones(mongo_connector)
# For each tracked TLD
for zone in zones:
logger.debug("Checking " + zone)
results = vt_instance.get_domain_report(zone)
if results is None:
logger.warning("Error querying zone " + zone)
elif results["response_code"] == -1:
logger.warning("VT unhappy with " + zone)
elif results["response_code"] == 0:
logger.warning("VT doesn't have " + zone)
else:
logger.debug("Matched " + zone)
results["zone"] = zone
results["created"] = datetime.now()
# Mongo doesn't allow key names with periods in them
# Re-assign to an undotted key name
if "Dr.Web category" in results:
results["Dr Web category"] = results.pop("Dr.Web category")
elif "alphaMountain.ai category" in results:
results["alphaMountain_ai category"] = results.pop(
"alphaMountain.ai category")
vt_collection.delete_one({"zone": zone})
if "last_https_certificate" in results:
if "extensions" in results["last_https_certificate"]:
if ("1.3.6.1.4.1.11129.2.4.2" in
results["last_https_certificate"]["extensions"]):
results["last_https_certificate"]["extensions"][
"sct_list"] = results["last_https_certificate"][
"extensions"].pop("1.3.6.1.4.1.11129.2.4.2")
mongo_connector.perform_insert(vt_collection, results)
# This sleep command is so that we don't exceed the daily limit on the free API
# This setting results in this script taking several days to complete
time.sleep(25)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Beging Main...
"""
global global_exit_flag
global global_retest_list
global global_sleep_time
global global_queue_size
global global_zgrab_path
logger = LoggingUtil.create_log(__name__)
global_retest_list = []
parser = argparse.ArgumentParser(description='Launch zgrab against IPs using port 22, 25, 443, or 465.')
parser.add_argument('-p', choices=['22', '25', '443', '465'], metavar="port", help='The port to scan: 22, 25, 443, or 465')
parser.add_argument('-t', default=5, type=int, metavar="threadCount", help='The number of threads')
parser.add_argument('--mx', action="store_true", help='Scan only IPs from MX records. Useful for SMTP scans.')
parser.add_argument('-s', default=0, type=int, metavar="sleepTime", help='Sleep time in order to spread out the batches')
parser.add_argument('--qs', default=0, type=int, metavar="queueSize", help='How many hosts to scan in a batch')
parser.add_argument('--zones_only', action="store_true", help='Scan only IPs from IP zones.')
parser.add_argument('--zgrab_path', default=global_zgrab_path, metavar='zgrabVersion', help='The version of ZGrab to use')
args = parser.parse_args()
if args.p == None:
logger.error("A port value (22, 25, 443, or 465) must be provided.")
exit(1)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
now = datetime.now()
logger.warning(str(now) + ": I am already running! Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
rm_connector = RemoteMongoConnector.RemoteMongoConnector()
all_dns_collection = rm_connector.get_all_dns_connection()
ip_manager = IPManager.IPManager(rm_connector, True)
jobs_manager = JobsManager.JobsManager(rm_connector, "zgrab_port_ip-" + args.p)
jobs_manager.record_job_start()
zones_struct = {}
zones_struct['zones'] = ZoneManager.get_distinct_zones(rm_connector)
# Not pretty but works
zones_struct['ip_manager'] = ip_manager
if args.mx:
(ips, ip_context) = get_mx_ips(zones_struct['zones'], ip_manager, all_dns_collection)
elif args.zones_only:
(ips, ip_context) = get_only_ipzones(ip_manager.Tracked_CIDRs)
else:
(ips, ip_context) = get_ips(ip_manager, all_dns_collection)
if args.s and int(args.s) > 0:
global_sleep_time = int(args.s)
if args.qs and int(args.qs) > 0:
global_queue_size = int(args.qs)
logger.info("Got IPs: " + str(len(ips)))
zones_struct['ip_context'] = ip_context
zgrab_collection = rm_connector.get_zgrab_port_data_connection()
if args.p == "443":
run_command = run_port_443_command
elif args.p == "22":
run_command = run_port_22_command
elif args.p == "25":
run_command = run_port_25_command
elif args.p == "465":
run_command = run_port_465_command
check_save_location("./json_p" + args.p)
global_zgrab_path = args.zgrab_path
threads = []
logger.debug("Creating " + str(args.t) + " threads")
for thread_id in range (1, args.t + 1):
thread = ZgrabThread(thread_id, global_work_queue, args.p, run_command, zones_struct, zgrab_collection)
thread.start()
threads.append(thread)
thread_id += 1
logger.info("Populating Queue")
global_queue_lock.acquire()
for ip in ips:
global_work_queue.put(ip)
global_queue_lock.release()
# Wait for queue to empty
while not global_work_queue.empty():
pass
# Notify threads it's time to exit
global_exit_flag = 1
# Wait for all threads to complete
for t in threads:
t.join()
logger.info ("Exiting Main Thread")
logger.info ("Global retest list: " + str(len(global_retest_list)))
# Retest any SMTP hosts that did not respond to the StartTLS handshake
if args.p == "25" and len(global_retest_list) > 0:
process_thread(logger, global_retest_list, args.p, run_port_25_no_tls_command, zones_struct, zgrab_collection, "retest")
# Remove old entries from before the scan
if args.p == "443":
other_results = zgrab_collection.find({'data.tls': {"$exists": True}, 'data.tls.timestamp': {"$lt": now}})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.tls': ""}})
elif args.p == "22":
if 'zgrab2' in global_zgrab_path:
other_results = zgrab_collection.find({'data.ssh': {"$exists": True}, 'data.ssh.timestamp': {"$lt": now}})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.ssh': ""}})
else:
other_results = zgrab_collection.find({'data.xssh': {"$exists": True}, 'data.xssh.timestamp': {"$lt": now}})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.xssh': ""}})
elif args.p == "25":
other_results = zgrab_collection.find({'data.smtp': {"$exists": True}, 'data.smtp.timestamp': {"$lt": now}})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.smtp': ""}})
elif args.p == "465":
other_results = zgrab_collection.find({'data.smtps': {"$exists": True}, 'data.smtps.timestamp': {"$lt": now}})
for result in other_results:
zgrab_collection.update_one({"_id": ObjectId(result['_id'])}, {"$unset": {'data.smtps': ""}})
# Remove any completely empty entries
zgrab_collection.remove({'data': {}})
jobs_manager.record_job_complete()
now = datetime.now()
print("Complete: " + str(now))
logger.info("Complete.")
def main():
"""
Begin Main...
"""
now = datetime.now()
print("Starting: " + str(now))
mongo_connector = MongoConnector.MongoConnector()
dns_manager = DNSManager.DNSManager(mongo_connector)
jobs_manager = JobsManager.JobsManager(mongo_connector, 'sonar_round_two')
google_dns = GoogleDNS.GoogleDNS()
jobs_manager.record_job_start()
zones = ZoneManager.get_distinct_zones(mongo_connector)
results = dns_manager.find_multiple({'type': 'cname'}, "sonar_dns")
round_two = []
round_three = []
# Get all the CNAME values from all_dns and append them to round_two
for result in results:
if is_tracked_zone(result['value'], zones):
round_two.append(result['value'])
print("Round two pre-list: " + str(len(round_two)))
dead_dns_collection = mongo_connector.get_dead_dns_connection()
for value in round_two:
is_present = dns_manager.find_count({'fqdn': value}, "sonar_dns")
if is_present == 0:
print(value + " not found")
time.sleep(1)
result = google_dns.fetch_DNS_records(value)
if result == []:
print("Unable to resolve")
original_records = dns_manager.find_multiple({"value": value},
"sonar_dns")
for record in original_records:
check = dead_dns_collection.find({
'fqdn': record['fqdn']
}).count()
if check == 0:
record.pop("_id")
dead_dns_collection.insert(record)
else:
for entry in result:
if is_tracked_zone(entry['fqdn'], zones):
new_record = entry
new_record['status'] = 'unconfirmed'
new_record['zone'] = get_fld_from_value(value, '')
new_record['created'] = datetime.now()
if result[0]['type'] == "cname" and is_tracked_zone(
entry['value'], zones):
add_to_list(entry['value'], round_three)
print("Found: " + value)
if new_record['zone'] != '':
dns_manager.insert_record(new_record, "marinus")
# For each tracked CName result found in the first pass across Sonar DNS
print("Round Three length: " + str(len(round_three)))
for hostname in round_three:
zone = get_fld_from_value(hostname, '')
if zone != None and zone != '':
ips = google_dns.fetch_DNS_records(hostname)
time.sleep(1)
if ips != []:
for ip_addr in ips:
if is_tracked_zone(ip_addr['fqdn'], zones):
record = {"fqdn": ip_addr['fqdn']}
record['zone'] = get_fld_from_value(
ip_addr['fqdn'], '')
record['created'] = datetime.now()
record['type'] = ip_addr['type']
record['value'] = ip_addr['value']
record['status'] = 'unconfirmed'
dns_manager.insert_record(new_record, "marinus")
else:
original_record = dns_manager.find_one({"fqdn": hostname},
"marinus")
if original_record != None:
original_record.pop("_id")
dead_dns_collection.insert(original_record)
print("Failed IP Lookup for: " + hostname)
else:
print("Failed match on zone for: " + hostname)
# Record status
jobs_manager.record_job_complete()
now = datetime.now()
print("Ending: " + str(now))
def main():
"""
Begin main...
"""
logger = LoggingUtil.create_log(__name__)
if is_running("get_censys_files.py"):
"""
Check to see if a download is in process...
"""
logger.warning("Can't run due to get_files running. Goodbye!")
exit(0)
if is_running(os.path.basename(__file__)):
"""
Check to see if a previous attempt to parse is still running...
"""
logger.warning("I am already running! Goodbye!")
exit(0)
# Make the relevant database connections
RMC = RemoteMongoConnector.RemoteMongoConnector()
ip_manager = IPManager.IPManager(RMC)
# Verify that the get_files script has a recent file in need of parsing.
jobs_collection = RMC.get_jobs_connection()
status = jobs_collection.find_one({'job_name': 'censys'})
if status['status'] != "DOWNLOADED":
logger.warning("The status is not set to DOWNLOADED. Goodbye!")
exit(0)
now = datetime.now()
print("Starting: " + str(now))
logger.info("Starting...")
# Collect the list of available zones
zones = ZoneManager.get_distinct_zones(RMC)
logger.info("Zones: " + str(len(zones)))
# Get the current configuration information for Marinus.
config_collection = RMC.get_config_connection()
configs = config_collection.find({})
orgs = []
for org in configs[0]['SSL_Orgs']:
orgs.append(org)
logger.info("Orgs: " + str(len(orgs)))
# Obtain the name of the decompressed file.
filename_f = open(FILENAME_FILE, "r")
decompressed_file = filename_f.readline()
filename_f.close()
# For manual testing: decompressed_file = "ipv4.json"
logger.info("Beginning file processing...")
# Remove old results from the database
results_collection = RMC.get_results_connection()
results_collection.remove({})
all_dns_collection = RMC.get_all_dns_connection()
try:
with open(decompressed_file, "r") as dec_f:
for line in dec_f:
try:
entry = json.loads(line)
"""
Does the SSL certificate match a known organization?
Is the IP address in a known CIDR?
Is the IP address recorded in Splunk?
"""
if check_in_org(entry, orgs) or \
ip_manager.is_tracked_ip(entry['ip']) or \
ip_manager.find_splunk_data(entry['ip'], "AWS") is not None or \
ip_manager.find_splunk_data(entry['ip'], "AZURE") is not None:
entry['zones'] = check_in_zone(entry, zones)
entry['aws'] = ip_manager.is_aws_ip(entry['ip'])
entry['azure'] = ip_manager.is_azure_ip(entry['ip'])
(domains,
zones) = lookup_domain(entry, zones,
all_dns_collection)
if len(domains) > 0:
entry['domains'] = domains
if len(zones) > 0:
for zone in zones:
if zone not in entry['zones']:
entry['zones'].append(zone)
insert_result(entry, results_collection)
# else:
# #This will add days to the amount of time necessary to scan the file.
# matched_zones = check_in_zone(entry, zones)
# if matched_zones != []:
# entry['zones'] = matched_zones
# entry['aws'] = ip_manager.is_aws_ip(entry['ip'])
# entry['azure'] = ip_manager.is_azure_ip(entry['ip'])
# insert_result(entry, results_collection)
except ValueError as err:
logger.error("Value Error!")
logger.error(str(err))
except:
logger.error("Line unexpected error: " +
str(sys.exc_info()[0]))
logger.error("Line unexpected error: " +
str(sys.exc_info()[1]))
except IOError as err:
logger.error("I/O error({0}): {1}".format(err.errno, err.strerror))
exit(1)
except:
logger.error("Unexpected error: " + str(sys.exc_info()[0]))
logger.error("Unexpected error: " + str(sys.exc_info()[1]))
exit(1)
# Indicate that the processing of the job is complete and ready for download to Marinus
jobs_collection.update_one({'job_name': 'censys'}, {
'$currentDate': {
"updated": True
},
"$set": {
'status': 'COMPLETE'
}
})
now = datetime.now()
print("Ending: " + str(now))
logger.info("Complete.")
