def __init__(self, hphosts_feed, output_file, config):
self.hphosts_feed = hphosts_feed
self.output_file = output_file
self.output_file_handler = False
self.config = config
self.resolver = DnsResolver()
self.csv_writer = False
self.resolvers = config.confvalues
self.resolver_names = config.confvalues.keys()
self.statistics = {}
def __init__(self, hphosts_feed, output_file, config):
self.hphosts_feed = hphosts_feed
self.output_file = output_file
self.output_file_handler = False
self.config = config
self.resolver = DnsResolver(retry_servfail=True)
self.csv_writer = False
self.resolvers = config.confvalues
self.resolver_names = config.confvalues.keys()
self.statistics = {}
# Initialise stats dict
for item in self.resolver_names:
self.statistics[item] = 0
def generate_results(self,domain):
result = {}
result['Domain'] = domain
# Iterate through the third-party DNS services
for resolver_name in self.resolver_names:
try:
dns_resolvers = self.resolvers[resolver_name]['resolvers']
domain = result['Domain']
resolver = DnsResolver(dns_resolvers, single_resolver=True)
# Retrieve the IP addresses that the third-party DNS service resolves
ip_addresses = resolver.get_ip_address(domain)
except Exception as e:
# Write DNS lookup error message in the CSV file
result[resolver_name] = str(e)
else:
blockpages = self.resolvers[resolver_name]['blockpages']
result[resolver_name] = self.generate_result(ip_addresses, blockpages, resolver_name, domain)
return (result)
def write_results(self, entries_to_process):
'''Write results into CSV file'''
counter = 1
# Create progress bar
bar = ProgressBar(entries_to_process, max_width=72)
# Write CSV header
csv_writer = self.open_csv_file()
# Iter through each feed entry from the hpHosts feed
for feed_entry in self.hphosts_feed.entries:
# Stop processing if the number of entries are higher than in '-n'
if counter > entries_to_process:
break
result = {}
# Update progress bar
bar.numerator = counter
print(bar, end='\r')
# Write phishing site details into CSV
result['Phishing Site Domain'] = feed_entry.title
result['Added to hpHosts'] = parse(feed_entry.published)
result['Phishing Site IP Address'] = re.findall(
r'[0-9]+(?:\.[0-9]+){3}', feed_entry.summary)[0]
# Iterate through the third-party DNS services
for resolver_name in self.resolver_names:
try:
dns_resolvers = self.resolvers[resolver_name]['resolvers']
phishing_domain = result['Phishing Site Domain']
resolver = DnsResolver(dns_resolvers)
# Retrieve the IP addresses that the third-party DNS service resolves
ip_addresses = resolver.get_ip_address(phishing_domain)
except Exception as e:
# Write DNS lookup error message in the CSV file
result[resolver_name] = e
else:
blockpages = self.resolvers[resolver_name]['blockpages']
result[resolver_name] = self.generate_result(
ip_addresses, blockpages, resolver_name)
# Write results into file
csv_writer.writerow(result)
# Flush file after writing each line
self.output_file_handler.flush()
counter += 1
# Close output file
self.output_file_handler.close()
return counter
class ConfParse(object):
'''ConfParse class'''
def __init__(self, file_name="conf/services.ini"):
self.file_name = file_name
self.config_parser = False
self.config = {}
self.dns_resolver = DnsResolver()
self.main()
def main(self):
self.config_parser = configparser.ConfigParser()
self.config_parser.read_file(open(self.file_name))
for entry in self.config_parser.sections():
resolvers = self.config_parser[entry]['resolvers']
blockpages = self.config_parser[entry]['blockpages']
self.config[entry] = {
'resolvers': self.parse_resolvers(resolvers),
'blockpages': self.parse_blockpages(blockpages)
}
def parse_resolvers(self, resolvers):
'''Parses IP addresses from the configuration file'''
# TODO: Change this to IPSet()
result = []
# Strip whitespaces
resolvers_list = [x.strip() for x in resolvers.split(',')]
# Validate IP addresses
for ip_address in resolvers_list:
# If the IP is valid, add to the results
if ipv4(ip_address):
result.append(ip_address)
else:
raise Exception(
"Invalid IP address in configuration file: {}".format(
ip_address))
return result
def parse_blockpages(self, blockpages):
'''
Parses 'blockpages =' entry from the configuration file
Get the IP address or domain name from the config file of the blocked pages
and add them into an IPSet.
'''
result = IPSet()
# Split entries from the config file and strip whitespaces
blockpages_list = [x.strip() for x in blockpages.split(',')]
# Process the IP addresses and domains
for blockpage in blockpages_list:
# If the DNS server blocks the query with NXDOMAIN
if blockpage == 'NXDOMAIN':
# Represent NXDOMAIN with 255.255.255.255/32
result.add('255.255.255.255/32')
# If the block page is hosted on a simple IP address
elif ipv4(blockpage):
result.add(blockpage)
# If the block page is hosted somewhere on a subnet
elif re.match('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(?:/\d{1,2}|)',
blockpage):
result.add(blockpage)
# If the block page is hosted under a CNAME record
elif domain(blockpage):
try:
result = self.dns_resolver.get_ip_address(blockpage)
except Exception as e:
print(
"\nDNS resolution error while retrieving list of block pages: {}\n"
.format(e))
exit(1)
# If the 'blockpage = ' has a 'none' value (e.g. Google DNS doesn't have a block page)
elif blockpage == 'none':
pass
# Throw exception if an invalid entry is found
else:
raise Exception("Invalid entry found: {}".format(blockpage))
return result
@property
def confvalues(self):
return self.config
def __init__(self, file_name="conf/services.ini"):
self.file_name = file_name
self.config_parser = False
self.config = {}
self.dns_resolver = DnsResolver()
self.main()
class Report(object):
'''Report class'''
def __init__(self, hphosts_feed, output_file, config):
self.hphosts_feed = hphosts_feed
self.output_file = output_file
self.output_file_handler = False
self.config = config
self.resolver = DnsResolver(retry_servfail=True)
self.csv_writer = False
self.resolvers = config.confvalues
self.resolver_names = config.confvalues.keys()
self.statistics = {}
# Initialise stats dict
for item in self.resolver_names:
self.statistics[item] = 0
def is_blocked(self, ip_addresses, blockpages, phishing_domain):
'''Verifies whether the IP address is on the blocked page list'''
# TODO: blockpages should be generated from self.config
intersection = ip_addresses & blockpages
# If response was NXDOMAIN, we need to verify if a non-filtering server respond with NXDOMAIN too
# e.g. the domain is taken down
if IPAddress('255.255.255.255') in ip_addresses:
try:
# Get the IP address from Google DNS
google_dns_response = self.resolver.get_ip_address(phishing_domain)
nx_intersection = ip_addresses & google_dns_response
# If Google DNS and the DNS server responds with different results, the
# blocking was unsuccessful
if len(nx_intersection) == 0:
response = 'NXDOMAIN'
return response
# If Google DNS and the DNS server under inspection responds with the same IP addresses
# the block is unsuccessful
else:
return False
# If response is NXDOMAIN from Google DNS, or any other error, return False
# which means we could not determine whether the site was blocked or not
except:
return False
elif len(intersection) > 0:
# If the IP address is in the list of IP addresses of block pages
# the website is blocked successfully
return intersection
# Return 'non-blocked' in case of any other errors
else:
return False
def generate_result(self, ip_addresses, blockpages, resolver_name, phishing_domain):
"""
Generates cell content in the CSV file
"""
result = False
# Return 'SITE_BLOCKED_OK' if the phishing site's domain name resolves to
# one of the block pages of the DNS services.
if self.is_blocked(ip_addresses, blockpages, phishing_domain):
result = 'SITE_BLOCKED_OK'
self.add_to_stats(resolver_name)
# If the website is not blocked, return with the website's IP address
else:
results = []
for ip_address in ip_addresses:
results.append(str(ip_address))
result = "\n".join(results)
return result
def open_csv_file(self):
'''Open CSV file and add header'''
try:
self.output_file_handler = open(self.output_file, 'w')
except Exception as e:
print("\n\nError opening output file {}: {}\n".format(args.o, e))
exit(1)
csv_header_fieldnames = [
'Added to hpHosts',
'Phishing Site Domain',
'Phishing Site IP Address'
]
csv_header_fieldnames.extend(sorted(self.resolver_names))
csv_writer = csv.DictWriter(self.output_file_handler, delimiter=',', fieldnames=csv_header_fieldnames)
csv_writer.writeheader()
return csv_writer
def write_results(self, entries_to_process):
'''Write results into CSV file'''
counter = 1
# Create progress bar
bar = ProgressBar(entries_to_process, max_width=72)
# Write CSV header
csv_writer = self.open_csv_file()
# Iter through each feed entry from the hpHosts feed
for feed_entry in self.hphosts_feed.entries:
# Stop processing if the number of entries are higher than in '-n'
if counter > entries_to_process:
break
result = {}
# Update progress bar
bar.numerator = counter
print(bar, end='\r')
# Write phishing site details into CSV
result['Phishing Site Domain'] = feed_entry.title
result['Added to hpHosts'] = parse(feed_entry.published)
result['Phishing Site IP Address'] = re.findall(r'[0-9]+(?:\.[0-9]+){3}', feed_entry.summary)[0]
# Iterate through the third-party DNS services
for resolver_name in self.resolver_names:
try:
dns_resolvers = self.resolvers[resolver_name]['resolvers']
phishing_domain = result['Phishing Site Domain']
resolver = DnsResolver(dns_resolvers, single_resolver=True)
# Retrieve the IP addresses that the third-party DNS service resolves
ip_addresses = resolver.get_ip_address(phishing_domain)
except Exception as e:
# Write DNS lookup error message in the CSV file
result[resolver_name] = e
else:
blockpages = self.resolvers[resolver_name]['blockpages']
result[resolver_name] = self.generate_result(ip_addresses, blockpages, resolver_name, phishing_domain)
# Write results into file
csv_writer.writerow(result)
# Flush file after writing each line
self.output_file_handler.flush()
counter += 1
# Close output file
self.output_file_handler.close()
return counter
def add_to_stats(self, resolver_name):
self.statistics[resolver_name] += 1
def print_stats_diagram(self, total_entries):
data = []
graph = Pyasciigraph(separator_length=4)
for resolver_name in sorted(self.resolver_names):
item = (resolver_name, self.statistics[resolver_name])
data.append(item)
item = ('TOTAL', total_entries)
data.append(item)
for line in graph.graph('Blocking Statistics:', data):
print(line)
class Report(object):
'''Report class'''
def __init__(self, domains, output_file, config):
self.domains = domains
self.output_file = output_file
self.output_file_handler = False
self.config = config
self.resolver = DnsResolver(retry_servfail=True)
self.csv_writer = False
self.resolvers = config.confvalues
self.resolver_names = config.confvalues.keys()
self.statistics = {}
# Initialise stats dict
for item in self.resolver_names:
self.statistics[item] = 0
def is_blocked(self, ip_addresses, blockpages, phishing_domain):
'''Verifies whether the IP address is on the blocked page list'''
# TODO: blockpages should be generated from self.config
intersection = ip_addresses & blockpages
# If response was NXDOMAIN, we need to verify if a non-filtering server respond with NXDOMAIN too
# e.g. the domain is taken down
# Answer is NXDomain. Let's investigate if it is due to blocking or the domain does not actually exist.
if IPAddress('255.255.255.255') in ip_addresses and IPAddress('255.255.255.255') in blockpages:
try:
# Get the IP address from Google DNS
google_dns_response = self.resolver.get_ip_address(phishing_domain)
nx_intersection = ip_addresses & google_dns_response
# If Google DNS and the DNS server responds with different results, the
# blocking was successful.
if len(nx_intersection) == 0:
response = 'NXDOMAIN'
#return response
return True
# If Google DNS and the DNS server under inspection responds with the same IP addresses
# the block is unsuccessful
else:
return False
# If response is NXDOMAIN from Google DNS, or any other error, return False
# which means we could not determine whether the site was blocked or not
except:
return False
elif len(intersection) > 0:
# If the IP address is in the list of IP addresses of block pages
# the website is blocked successfully
return True
# return intersection
# Return 'non-blocked' in case of any other errors
else:
return False
def generate_result(self, ip_addresses, blockpages, resolver_name, phishing_domain):
"""
Generates cell content in the CSV file
"""
result = False
# Return 'SITE_BLOCKED_OK' if the phishing site's domain name resolves to
# one of the block pages of the DNS services.
blocked = self.is_blocked(ip_addresses, blockpages, phishing_domain)
if blocked:
result = 'SITE_BLOCKED_OK' +"(" + ",".join(str(answer).replace("255.255.255.255","NXDOMAIN") for answer in ip_addresses) +")"
# self.add_to_stats(resolver_name)
# If the website is not blocked, return with the website's IP address
else:
results = []
for ip_address in ip_addresses:
results.append(str(ip_address).replace("255.255.255.255","NXDOMAIN") )
result = ",".join(results)
return result
def open_csv_file(self):
'''Open CSV file and add header'''
if not os.path.exists('output'):
os.makedirs('output')
try:
self.output_file_handler = open("output/" + self.output_file +".csv", 'w')
except Exception as e:
print("\n\nError opening output file {}: {}\n".format(args.o, e))
exit(1)
csv_header_fieldnames = ["Domain"]
csv_header_fieldnames.extend(sorted(self.resolver_names))
csv_writer = csv.DictWriter(self.output_file_handler, delimiter=',', fieldnames=csv_header_fieldnames)
csv_writer.writeheader()
return csv_writer
def generate_results(self,domain):
result = {}
result['Domain'] = domain
# Iterate through the third-party DNS services
for resolver_name in self.resolver_names:
try:
dns_resolvers = self.resolvers[resolver_name]['resolvers']
domain = result['Domain']
resolver = DnsResolver(dns_resolvers, single_resolver=True)
# Retrieve the IP addresses that the third-party DNS service resolves
ip_addresses = resolver.get_ip_address(domain)
except Exception as e:
# Write DNS lookup error message in the CSV file
result[resolver_name] = str(e)
else:
blockpages = self.resolvers[resolver_name]['blockpages']
result[resolver_name] = self.generate_result(ip_addresses, blockpages, resolver_name, domain)
return (result)
def write_results(self,thread_count,domain_queue):
'''Write results into CSV file'''
counter = 1
workload(self,thread_count,domain_queue)
# Write CSV header
csv_writer = self.open_csv_file()
with open(("output/" + self.output_file+".json"), 'w') as f:
while not (output_queue.empty()):
row = output_queue.get()
csv_writer.writerow(row)
f.write(json.dumps(row)+"\n")
for resolver in row:
if ("SITE_BLOCKED_OK" in str(row[resolver])):
self.add_to_stats(resolver)
# Flush file after writing each line
self.output_file_handler.flush()
# Close output file
self.output_file_handler.close()
def add_to_stats(self, resolver_name):
self.statistics[resolver_name] += 1
def print_stats_diagram(self, total_entries):
data = []
graph = Pyasciigraph(separator_length=4)
for resolver_name in sorted(self.resolver_names):
item = (resolver_name, self.statistics[resolver_name])
data.append(item)
item = ('TOTAL', total_entries)
data.append(item)
for line in graph.graph('Blocking Statistics:', data):
print(line)