def main():
publicdns_url = 'https://public-dns.info/nameservers.csv'
publicdns_file = 'public-dns-nameservers.csv'
download_to_file(publicdns_url, publicdns_file)
dnscrypt_url = "https://raw.githubusercontent.com/DNSCrypt/dnscrypt-resolvers/master/v1/dnscrypt-resolvers.csv"
dnscrypt_file = "dnscrypt-resolvers.csv"
download_to_file(dnscrypt_url, dnscrypt_file)
ipv4, ipv6, hostname = get_lists_publidns(publicdns_file)
ipv4_c, ipv6_c = get_lists_dnscrypt(dnscrypt_file)
ipv4 += ipv4_c
ipv6 += ipv6_c
process(ipv4, ipv6, hostname)
def process(file, dst):
with open(get_abspath_source_file(file), newline='\n', encoding='utf-8', errors='replace') as csv_file:
sites = csv_file.readlines()[:10000]
warninglist = {
'name': 'Top 10K websites from Majestic Million',
'version': get_version(),
'description': 'Event contains one or more entries from the top 10K of the most used websites (Majestic Million).',
'matching_attributes': ['hostname', 'domain'],
'type': 'hostname',
'list': []
}
for site in sites:
v = site.split(',')[2]
warninglist['list'].append(v.rstrip())
write_to_file(warninglist, dst)
if __name__ == '__main__':
majestic_url = 'http://downloads.majestic.com/majestic_million.csv'
majestic_file = 'majestic_million.csv'
majestic_dst = 'majestic_million'
download_to_file(majestic_url, majestic_file)
process(majestic_file, majestic_dst)
'description':
"List of known Cloudflare IP ranges (https://www.cloudflare.com/ips/)",
'type': "cidr",
'list': [],
'matching_attributes': ["ip-dst", "ip-src", "domain|ip"]
}
for file in files:
with open(file, 'r') as f:
ips = f.readlines()
for ip in ips:
warninglist['list'].append(ip.strip())
write_to_file(warninglist, dst)
if __name__ == '__main__':
cf_base_url = "https://www.cloudflare.com/"
uri_list = ['ips-v4', 'ips-v6']
cf_dst = 'cloudflare'
to_process = list()
for uri in uri_list:
url = cf_base_url + uri
file = 'cloudflare_{}.txt'.format(uri)
download_to_file(url, file)
to_process.append(file)
process(to_process, cf_dst)
servers_list = csv.reader(csv_file, delimiter=',', quotechar='"')
lipv4 = []
lipv6 = []
lhostname = []
for row in servers_list:
if row[7] in (None, ""):
try:
ip = ipaddress.ip_address(row[0])
if ip.version == 4:
lipv4.append(ip.compressed)
elif ip.version == 6:
lipv6.append(ip.compressed)
if row[1] not in (None, "", '.'):
lhostname.append(row[1])
except ValueError as exc:
logging.warning(str(exc))
return lipv4, lipv6, lhostname
if __name__ == '__main__':
publicdns_url = 'https://public-dns.info/nameservers.csv'
publicdns_file = 'public-dns-nameservers.csv'
download_to_file(publicdns_url, publicdns_file)
process(publicdns_file)
for site in sites:
v = site.decode('UTF-8').split(',')[1]
warninglist['list'].append(v.strip().replace('\\r\\n', ''))
write_to_file(warninglist, dst)
def get_lists(file):
with zipfile.ZipFile(get_abspath_source_file(file), 'r') as cisco_lists:
for name in cisco_lists.namelist():
if name == "top-1m.csv":
with cisco_lists.open(name) as cisco_list:
all = cisco_list.readlines()
top1k = all[:1000]
top5k = all[:5000]
top10k = all[:10000]
top20k = all[:20000]
else:
continue
return top1k, top5k, top10k, top20k
if __name__ == '__main__':
cisco_url = "http://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip"
cisco_file = "cisco_top-1m.csv.zip"
download_to_file(cisco_url, cisco_file)
process(cisco_file)
]
warninglist['list'] = []
for site in sites:
v = site.decode('UTF-8').split(',')[1]
warninglist['list'].append(v.rstrip())
write_to_file(warninglist, dst)
def get_lists(file):
with zipfile.ZipFile(get_abspath_source_file(file), 'r') as tranco_lists:
for name in tranco_lists.namelist():
if name == 'top-1m.csv':
with tranco_lists.open(name) as tranco:
all_sites = tranco.readlines()
top10k = all_sites[:10000]
else:
continue
return top10k, all_sites
if __name__ == '__main__':
tranco_url = 'https://tranco-list.eu/top-1m.csv.zip'
tranco_file = 'tranco_top-1m.csv.zip'
download_to_file(tranco_url, tranco_file)
process(tranco_file)
with open(get_abspath_source_file(file), 'r') as json_file:
amazon_aws_ip_list = json.load(json_file)
l = []
for prefix in amazon_aws_ip_list['prefixes']:
l.append(prefix['ip_prefix'])
for prefix in amazon_aws_ip_list['ipv6_prefixes']:
l.append(prefix['ipv6_prefix'])
warninglist = {
'name': 'List of known Amazon AWS IP address ranges',
'version': get_version(),
'description':
'Amazon AWS IP address ranges (https://ip-ranges.amazonaws.com/ip-ranges.json)',
'type': 'cidr',
'list': l,
'matching_attributes': ["ip-src", "ip-dst", "domain|ip"]
}
write_to_file(warninglist, dst)
if __name__ == '__main__':
amazon_url = "https://ip-ranges.amazonaws.com/ip-ranges.json"
amazon_file = "amazon_ip-ranges.json"
amazon_dst = "amazon-aws"
download_to_file(amazon_url, amazon_file)
process(amazon_file, amazon_dst)
'name': "Top 500 domains and pages from https://moz.com/top500",
'type': 'string',
'list': [],
'matching_attributes': ['hostname', 'domain', 'uri', 'url']
}
for file in files:
with open(get_abspath_source_file(file)) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
v = row[1]
warninglist['list'].append(v.rstrip().rstrip('/'))
write_to_file(warninglist, dst)
if __name__ == '__main__':
moz_domains_url = "https://moz.com/top-500/download/?table=top500Domains"
#moz_pages_url = "https://moz.com/top500/pages/csv"
moz_domains_file = "moz-top500.domains.csv"
#moz_pages_file = "moz-top500.pages.csv"
moz_dst = 'moz-top500'
download_to_file(moz_domains_url, moz_domains_file)
#download_to_file(moz_pages_url, moz_pages_file)
#process([moz_domains_file, moz_pages_file], moz_dst)
process([moz_domains_file], moz_dst)
}
for file in files:
with open(get_abspath_source_file(file), 'r') as f:
ips = f.readlines()
for ip in ips:
iptoadd = ip.strip()
try:
ipaddress.ip_network(ip.strip())
except ValueError as err: # if it's host given strip to the subnet
iptoadd = str(ipaddress.IPv6Interface(ip.strip()).ip)
warninglist['list'].append(iptoadd)
write_to_file(warninglist, dst)
if __name__ == '__main__':
# Base url where a text file is attached https://support.stackpath.com/hc/en-us/articles/360001091666-Whitelist-CDN-WAF-IP-Blocks"
sp_base_url = "https://support.stackpath.com/hc/en-us/article_attachments/360096407372/ipblocks.txt"
filename = 'ipblocks.txt'
sp_dst = 'stackpath'
to_process = list()
#url = get_file_link(sp_base_url, filename)
file = 'stackpath_{}'.format(filename)
download_to_file(sp_base_url, file)
to_process.append(file)
process(to_process, sp_dst)
for name in alexa_lists.namelist():
if name == "top-1m.csv":
with alexa_lists.open(name) as top:
top1000 = top.readlines()[:1000]
else:
continue
warninglist = {
'description': "Event contains one or more entries from the top 1000 of the most used website (Alexa).",
'version': get_version(),
'name': "Top 1000 website from Alexa",
'type': 'hostname',
'list': [],
'matching_attributes': ['hostname', 'domain', 'url', 'domain|ip']
}
for site in top1000:
v = site.decode('UTF-8').split(',')[1]
warninglist['list'].append(v.rstrip())
write_to_file(warninglist, dst)
if __name__ == "__main__":
alexa_url = "http://s3.amazonaws.com/alexa-static/top-1m.csv.zip"
alexa_file = "alexa_top-1m.csv.zip"
alexa_dst = "alexa"
download_to_file(alexa_url, alexa_file)
process(alexa_file, alexa_dst)
'description':
"Fingerprint of {type} taken from Mozilla's lists at https://wiki.mozilla.org/CA"
.format(type=type),
'list':
hashes,
'type':
'string',
'matching_attributes': [
"md5", "sha1", "sha256", "filename|md5", "filename|sha1",
"filename|sha256", "x509-fingerprint-md5", "x509-fingerprint-sha1",
"x509-fingerprint-sha256"
]
}
write_to_file(warninglist, dst)
if __name__ == '__main__':
Included_CA_url = 'https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReportPEMCSV'
Included_CA_file = 'IncludedCACertificateReportPEMCSV.csv'
Included_CA_dst = 'mozilla-CA'
CA_known_intermediate_url = 'https://ccadb-public.secure.force.com/mozilla/PublicAllIntermediateCertsWithPEMCSV'
CA_known_intermediate_file = 'PublicAllIntermediateCertsWithPEMCSV.csv'
CA_known_intermediate_dst = 'mozilla-IntermediateCA'
download_to_file(Included_CA_url, Included_CA_file)
process(Included_CA_file, Included_CA_dst, 'trusted CA certificates')
download_to_file(CA_known_intermediate_url, CA_known_intermediate_file)
process(CA_known_intermediate_file, CA_known_intermediate_dst,
'known intermedicate of trusted certificates')
def process(file, dst):
warninglist = {
'name': 'List of known Microsoft Azure Datacenter IP Ranges',
'version': get_version(),
'description': 'Microsoft Azure Datacenter IP Ranges',
'list': [],
'matching_attributes': ["ip-src", "ip-dst", "domain|ip"],
'type': 'cidr'
}
with open(file, 'r') as json_file:
ms_azure_ip_list = json.load(json_file)
for value in ms_azure_ip_list['values']:
warninglist['list'] += value['properties']['addressPrefixes']
write_to_file(warninglist, dst)
if __name__ == '__main__':
ms_azure_url = 'https://www.microsoft.com/en-us/download/confirmation.aspx?id=56519'
ms_azure_file = 'ms-azure.json'
ms_azure_dst = 'microsoft-azure'
ms_azure_json_url = get_json_url(download(ms_azure_url))
download_to_file(ms_azure_json_url, ms_azure_file)
process(ms_azure_file, ms_azure_dst)
}, {
"name":
"List of known Microsoft Azure US Government Cloud Datacenter IP Ranges",
"description":
"Microsoft Azure US Government Cloud Datacenter IP Ranges",
"url":
"https://www.microsoft.com/en-us/download/confirmation.aspx?id=57063",
"file": "ms-azure-us-gov.json",
"destination_folder": "microsoft-azure-us-gov",
}, {
"name": "List of known Microsoft Azure Germany Datacenter IP Ranges",
"description": "Microsoft Azure Germany Datacenter IP Ranges",
"url":
"https://www.microsoft.com/en-us/download/confirmation.aspx?id=57064",
"file": "ms-azure-germany.json",
"destination_folder": "microsoft-azure-germany",
}, {
"name": "List of known Microsoft Azure China Datacenter IP Ranges",
"description": "Microsoft Azure China Datacenter IP Ranges",
"url":
"https://www.microsoft.com/en-us/download/confirmation.aspx?id=57062",
"file": "ms-azure-china.json",
"destination_folder": "microsoft-azure-china",
}]
for type in TYPES:
ms_azure_json_url = get_json_url(download(type["url"]))
download_to_file(ms_azure_json_url, type["file"])
process(type["file"], type["destination_folder"], type["name"],
type["description"])
