def print_summary():
"""
Print stats from all evaluated categories in DB
"""
db.open_connection()
db.exec_query("SELECT category from results GROUP by category;")
table_sum = prettytable.PrettyTable(["Category", "Evaluated", "Meth 1", "Meth 2", "Meth 3", "Meth 4", "Hosted"])
categories = db.cur.fetchall()
for p in categories:
db.exec_query("SELECT * from results where `category`='{category}'".format(category=p[0]))
ec = EvaluatedCategory(db.cur.fetchall())
sums = ec.method_sums
percent_sums = ec.get_percents()
table_sum.add_row([
p[0],
ec.total,
"{} ({}%)".format(sums[1], percent_sums[1]),
"{} ({}%)".format(sums[2], percent_sums[2]),
"{} ({}%)".format(sums[3], percent_sums[3]),
"{} ({}%)".format(sums[4], percent_sums[4]),
"{} ({}%)".format(sums["total"], percent_sums["total"]),
])
print(table_sum)
db.close_connection()
def is_known_hoster(hoster):
db.open_connection()
db.exec_query(
"SELECT COUNT(*) from webhoster where hoster_domain='{url}'".format(
url=hoster))
return True if db.cur.fetchone()[0] else False
db.close_connection()
def print_category(category):
db.open_connection()
db.exec_query("SELECT * from results where `category`='{category}'".format(
category=category))
category_result = EvaluatedCategory(db.cur.fetchall())
print(category_result)
category_result.to_csv(file='export/' + category + '_export.csv')
db.close_connection()
def print_category(category):
"""
Function for printing category results to stdout.
:param category:
"""
db.open_connection()
db.exec_query("SELECT * from results where `category`='{category}'".format(
category=category))
# Creating object EvaluatedCategory, pass DB result to class constructor
category_result = EvaluatedCategory(db.cur.fetchall())
print(category_result)
print(category_result.get_stats())
db.close_connection()
"WHERE server_ip {} ".format(query))
else:
database.print_query_result(
"SELECT domain, server_ip as 'Server IP', server_ip_name as 'Server domain name' from results "
"WHERE server_ip {}"
"ORDER by domain".format(query))
print("Hosted pages:")
database.print_query_result(
"SELECT domain, category from results "
"WHERE server_ip LIKE '%{}%' "
"ORDER by domain".format(ip))
if __name__ == '__main__':
# Argument parsing section
parser = argparse.ArgumentParser(description='Print list of detected page on specified webserver. ')
parser.add_argument('query', help='IP/Network/webhoster query', type=str)
parser.add_argument('--summary', help='Print summary', action='store_true')
parser.add_argument('--webhoster', help='Print pages hosted on specified webhosting', action='store_true')
args = parser.parse_args()
database.open_connection()
if args.webhoster:
print_pages_webhoster(args.query, args.summary)
else:
print_pages_ip(args.query, args.summary)
database.close_connection()
def process_domain(domain: PreparedDomain, category: str, reevaluate: bool = False, restoreconn: bool = False,
cz_omit: bool = True, subdom_omit: bool = False,
output: object = None, db_conn: object = None) -> None:
"""
It will process given domain. The base evaluation function.
:param domain: Domain name
:param category: Category for this page
:param reevaluate: Reevaluate if found in db
:param restoreconn: It will call external script in case no WHOIS connectivity
:param cz_omit: It will omit domain within .cz domain zone
:param subdom_omit: It will omit domain on 3rd and higher level
:param output: Reference to opened file to write output log
:param db_conn: Reference to opened database connection
:return:
"""
if db_conn is None:
db.open_connection()
db_conn_created = True
else:
db.take_connection(db_conn)
db_conn_created = False
print("\n========================================================================\nProcessing page:", domain)
di = page_evaluator.get_domain_info(domain.domain)
if not di.get('serverIp'):
print("[ERROR][NO_DNS]", domain, 'Unable to translate domain. Skipping...')
return
if cz_omit:
if di.get('tld') != 'cz':
print("[NOTICE][NO_CZ]", domain, "omitted, not in CZ zone.")
return
if subdom_omit:
if di.get('subdomain').lower() not in ['', 'www', None]:
print("[NOTICE][SUBDOM]", domain, "is subdomain, omitted.")
return
if is_domain_evaluated(di.get('baseDomain')):
if reevaluate:
db.exec_query(delete_result.format(domain=di.get('baseDomain')))
db.commit()
else:
print("[NOTICE] Been evaluated in past:", domain, "skipping...")
return
whois_info = whois_parser.probe_domain(di['baseDomain'], restoreconn)
if whois_info is {}:
print("[ERROR][WHOIS_NIC] - no valid response for ", domain)
return
# Get WHOIS and GEO data about IP
try:
whois_ip_info = whois_parser.probe_ip(di['serverIp'])
except Exception as e:
print("[ERROR][WHOIS_IP] for ", di['serverIp'], e)
return
server_stats = wserver.probe_server(di['baseDomain'])
server_stats_https = wserver.probe_server(di['baseDomain'], True)
https_supported = check_https(server_stats_https)
if whois_info.get('holderId'):
contact_info = whois_parser.probe_contact(whois_info.get('holderId'), restoreconn)
else:
contact_info = {}
evaluated = page_evaluator.evaluate_domain(di, whois_info, whois_ip_info)
# Print given note
if domain.note:
print("[DOMAIN-NOTE]", domain.note)
print("[INFO][DOMAIN] b.dom: {bDomain}, s.IP: {IP}, s.IPv6 {IP6}, s.IP name: {IP_name}".format(
bDomain=di.get('baseDomain'),
IP=di.get('serverIp'),
IP6=di.get('serverIp6'),
IP_name=di.get('netDomainFull')))
if whois_info is not {}:
print("[INFO][CZNIC_WHOIS] holder: {hold}, NS: {nsset}".format(hold=whois_info.get('holderName'),
nsset=whois_info.get('nsset')))
if contact_info is not {}:
print("[INFO][CZNIC_CONTACT] email: {email}, ident: {ident}".format(email=contact_info.get('email'),
ident=contact_info.get('ident')))
print("[INFO][NET] email: {email}, org: {ident}, GEO {geo}".format(email=whois_ip_info.contact_email,
ident=whois_ip_info.get_organization(),
geo=whois_ip_info.geo_ip))
print("[INFO][WEBSRV] stat: {status}, sign: {sign}, HTTPS: {https_sup}".format(status=server_stats.get('status'),
sign=server_stats.get('server'),
https_sup=https_supported))
print("\n[EVAL][RESULTS] ", evaluated)
# Write to DB if was given category to store results
if category:
db.exec_query(add_result.format(
domain=di.get('baseDomain'),
res_rev=evaluated.method_reverse_dns,
res_known=evaluated.method_known_hoster,
res_whois=evaluated.method_whois,
res_email=evaluated.method_email,
s_server=server_stats.get('server'),
s_powerer=server_stats.get('x_powerer'),
s_type=server_stats.get('c_type'),
s_ip=di.get('serverIp'),
s_ip_name=di.get('netDomainFull'),
s_ip_domain=di.get('netDomain'),
dm_keyset=whois_info.get('keyset'),
dm_holdername=whois_info.get('holderName'),
ip_holdername=whois_ip_info.get_organization(),
category=category,
original_url=domain.original_url,
note=domain.note,
ipv6_addr=di.get('serverIp6'),
https_support=https_supported
))
db.commit()
if output is not None or False:
output.write("{dom}\t{m_wh}\t{m_rd}\t{m_h}\t{m_e}\t{s_ip}\t{d_hold}\t{ip_org}\t{ip_em}\n".format(
dom=di.get('domain'),
m_wh=EvaluatedPage.bool2switcher(evaluated.method_whois),
m_rd=EvaluatedPage.bool2switcher(evaluated.method_reverse_dns),
m_h=EvaluatedPage.bool2switcher(evaluated.method_known_hoster),
m_e=EvaluatedPage.bool2switcher(evaluated.method_email),
s_ip=di.get('serverIp'),
d_hold=whois_info.get('holderName'),
ip_org=whois_ip_info.get_organization(),
ip_em=whois_ip_info.contact_email
))
if db_conn_created is True:
db.close_connection()
db.exec_query(
"SELECT lat,lon,city,COUNT(lat) as count from webserver GROUP by lat,lon"
)
for e in db.cur.fetchall():
radius = e[3] * 5
folium.CircleMarker(
location=[e[0], e[1]],
popup="{} ({})".format(e[2], e[3]),
radius=15 if radius > 15 else radius).add_to(folium_map)
folium_map.save('visualisation/coords.html')
if __name__ == '__main__':
# Parse command line args
parser = argparse.ArgumentParser(
description='Provides some basic data operations ')
parser.add_argument('type',
help='Type of request',
type=str,
choices=['coords'])
args = parser.parse_args()
db.open_connection()
if args.type == 'coords':
coords_map()
db.close_connection()
def parse_file(filename,
url_col,
delimiter,
quotechar,
skiplines,
category,
restoreconn=False,
reevaluate=False,
cz_omit=True,
subdom_omit=False,
note_column=None):
db.open_connection() # create DB connection
total = 0
domains_unvisited = set()
# Read CSV stream from given file
with open(filename, 'r') as csvfile:
filereader = csv.reader(csvfile,
delimiter=delimiter,
quotechar=quotechar)
# Skip given count of lines
for _ in range(skiplines):
next(filereader)
# Do for each line in CSV file
for row in filereader:
url = row[url_col - 1]
# Selecting note
if note_column == -1: # save whole row
note = json.dumps(row)
elif note_column:
note = row[note_column - 1]
else:
note = None
# Check if domain name exists
if url not in [None, "", "NA"]:
domains_unvisited.add(PreparedDomain(url, note))
print("Unique domain for visiting: ")
# View the list of prepared objects
domains_unvisited_list = list()
for dom in domains_unvisited:
domains_unvisited_list.append(dom.domain)
print(', '.join(domains_unvisited_list))
with open("output.txt", "a") as output:
output.write(
"DOMAIN\tM_WHOIS\tZM_REV\tM_Znamy\tM_MAIL\tServer IP\tDrzitel\tKontakt\tEmail\n"
"---------------------------------------------------------------------------------\n"
)
for domain in domains_unvisited:
try:
process_page.process_domain(domain, category, reevaluate,
restoreconn, cz_omit, subdom_omit,
output)
total += 1
except (KeyboardInterrupt, SystemExit):
print("Exited by user.")
exit(0)
except RuntimeError:
# report error and proceed
print("[ERROR]", domain, " Unexcepted error:",
sys.exc_info()[0])
print("[STAT]: TOTAL " + repr(total) + "\n")
db.close_connection()