def getBlockedDomainsOld(connstr, collapse=True):
"""
Keeped there for comparison, or how to speed up python in 100 times
We don't need to block domains if the same wildcard domain is blocked
We don't need to block 3-level wildcard if 2-level wildcard is blocked
:param connstr: smth like "engine://*****:*****@host:port/dbname"
:param collapse: merge domains if wildcard analogue exists
:return: 2 sets: domains and wildcard domains
"""
bldt = BlockData(connstr)
domains = bldt.getBlockedResourcesSet('domain')
wdomains = bldt.getBlockedResourcesSet('domain-mask')
if not collapse:
return [list(domains), list(wdomains)]
# Dedupe wdomains
wds = wdomains.copy()
for wd in wds:
regex = re.compile('''^.+\.''' + wd + '''$''')
for wdom in wds:
if regex.fullmatch(wdom):
# Using discard to ignore redelete.
wdomains.discard(wdom)
# Dedupe domains with wdomains
for wd in wdomains.copy():
regex = re.compile('''^(.*[^.]\.)?''' + wd + '''$''')
for dom in domains.copy():
if regex.fullmatch(dom):
# Using discard to ignore redelete.
domains.discard(dom)
return [list(domains), list(wdomains)]
def getBlockedIPCount(connstr):
bldt = BlockData(connstr)
ips = [
ipaddress.ip_network(addr)
for addr in bldt.getBlockedResourcesSet('ip')
]
ipsubs = [
ipaddress.ip_network(addr)
for addr in bldt.getBlockedResourcesSet('ipsubnet')
]
ipNum = sum(
map(lambda x: x.num_addresses,
ipaddress.collapse_addresses(ips + ipsubs)))
return ipNum
def _getBlockedDataList(connstr, entityname):
"""
Function for debug purposes
:param connstr: smth like "engine://*****:*****@host:port/dbname"
:return: entities set
"""
return list(BlockData(connstr).getBlockedResourcesSet(entityname))
def getBlockedDomains(connstr, collapse=True):
"""
Brand new procedure. Uses domain tree to cleanup excess domains.
:param connstr: smth like "engine://*****:*****@host:port/dbname"
:param collapse: merge domains if wildcard analogue exists
:return: 2 sets: domains and wildcard domains
"""
bldt = BlockData(connstr)
domains = bldt.getBlockedResourcesSet('domain')
wdomains = bldt.getBlockedResourcesSet('domain-mask')
if not collapse:
return [list(domains), list(wdomains)]
# Building domains tree
dnstree = mkdnstree(domains, wdomains)
# Coalescing the tree to a list of domain-as-lists
# Starting with TLD, not 0LD
dnsmap = mapdnstree(dnstree[""])
# Making text domains and wdomains again
return list(map(list, dnslistmerged(dnsmap)))
def getBlockedIPsFromSubnets(connstr):
"""
Explodes restricted ip subnets into IP list.
:param connstr: smth like "engine://*****:*****@host:port/dbname"
:return: IP list.
"""
ipsubs = {
ipaddress.ip_network(addr)
for addr in BlockData(connstr).getBlockedResourcesSet('ipsubnet')
}
return [str(host) for ipsub in ipsubs for host in ipsub.hosts()]
def getBlockedIPs(connstr, collapse=True):
"""
Merges IPs into IP subnets containing first ones.
:param connstr: smth like "engine://*****:*****@host:port/dbname"
:param collapse: merge and minimize IPs and networks
:return: The total and the list of ip subnets, using /32 for ips.
"""
bldt = BlockData(connstr)
ips = [
ipaddress.ip_network(addr)
for addr in bldt.getBlockedResourcesSet('ip')
]
ipsubs = [
ipaddress.ip_network(addr)
for addr in bldt.getBlockedResourcesSet('ipsubnet')
]
ipsall = ips + ipsubs
if collapse:
ipsall = list(ipaddress.collapse_addresses(ipsall))
ipNum = sum(map(lambda x: x.num_addresses, ipsall))
return [list(map(str, ipsall)), ipNum]
def getBlockedIPList(connstr, collapse=True, ipv6=False):
"""
Complementary function for getting only IP list
:param connstr: smth like "engine://*****:*****@host:port/dbname"
:param collapse: merge and minimize IPs and networks
:param ipv6: use ipv6 entities
:return: The total and the list of ip subnets, using /32 for ips.
"""
bldt = BlockData(connstr)
if ipv6:
ips = [
ipaddress.ip_network(addr)
for addr in bldt.getBlockedResourcesSet('ipv6')
]
ipsubs = [
ipaddress.ip_network(addr)
for addr in bldt.getBlockedResourcesSet('ipv6subnet')
]
else:
ips = [
ipaddress.ip_network(addr)
for addr in bldt.getBlockedResourcesSet('ip')
]
ipsubs = [
ipaddress.ip_network(addr)
for addr in bldt.getBlockedResourcesSet('ipsubnet')
]
ipsall = ips + ipsubs
if collapse:
return list(ipaddress.collapse_addresses(ipsall))
return list(ipsall)
def getBlockedHTTPS(connstr):
"""
:param connstr: smth like "engine://*****:*****@host:port/dbname"
:return URLs strings list
"""
return list(BlockData(connstr).getBlockedResourcesSet('https'))