def print_summary():
"""
Print stats from all evaluated categories in DB
"""
db.open_connection()
db.exec_query("SELECT category from results GROUP by category;")
table_sum = prettytable.PrettyTable(["Category", "Evaluated", "Meth 1", "Meth 2", "Meth 3", "Meth 4", "Hosted"])
categories = db.cur.fetchall()
for p in categories:
db.exec_query("SELECT * from results where `category`='{category}'".format(category=p[0]))
ec = EvaluatedCategory(db.cur.fetchall())
sums = ec.method_sums
percent_sums = ec.get_percents()
table_sum.add_row([
p[0],
ec.total,
"{} ({}%)".format(sums[1], percent_sums[1]),
"{} ({}%)".format(sums[2], percent_sums[2]),
"{} ({}%)".format(sums[3], percent_sums[3]),
"{} ({}%)".format(sums[4], percent_sums[4]),
"{} ({}%)".format(sums["total"], percent_sums["total"]),
])
print(table_sum)
db.close_connection()
def is_known_hoster(hoster):
db.open_connection()
db.exec_query(
"SELECT COUNT(*) from webhoster where hoster_domain='{url}'".format(
url=hoster))
return True if db.cur.fetchone()[0] else False
db.close_connection()
def print_category(category):
db.open_connection()
db.exec_query("SELECT * from results where `category`='{category}'".format(
category=category))
category_result = EvaluatedCategory(db.cur.fetchall())
print(category_result)
category_result.to_csv(file='export/' + category + '_export.csv')
db.close_connection()
def is_domain_evaluated(domain):
"""
Detect if page has been evaluated in past.
:param domain: Extracted domain name
:return: Boolean value. True, if page is in db.
"""
db.exec_query("SELECT COUNT(*) from results where `domain`='{domain}'".format(domain=domain))
db_result = db.cur.fetchone()
if db_result is None:
return False
return True if db_result[0] else False
def to_db(self):
query = '''
INSERT INTO service_notifications(service, host, timestamp, text, level)
VALUES ('{0}', '{1}', '{2}', '{3}', '{4}')'''.format(
self.author, self.host, self.timestamp, self.text, self.level)
return db.exec_query(query)
def print_category(category):
"""
Function for printing category results to stdout.
:param category:
"""
db.open_connection()
db.exec_query("SELECT * from results where `category`='{category}'".format(
category=category))
# Creating object EvaluatedCategory, pass DB result to class constructor
category_result = EvaluatedCategory(db.cur.fetchall())
print(category_result)
print(category_result.get_stats())
db.close_connection()
def coords_map():
"""
Create map from DB results
:return:
"""
tileset = r'https://api.mapbox.com/styles/v1/mapbox/light-v9/tiles/256/{z}/{x}/{y}?access_token=pk.eyJ1IjoicGV0cmlsZ25lciIsImEiOiJjajI4cGlxbDgwMDVxMzJvanBldTlibWNrIn0.9W5sqnxpaPXlJe1VjlZaZA'
folium_map = folium.Map(location=[49.19, 16.60],
tiles=tileset,
attr='Coords')
db.exec_query(
"SELECT lat,lon,city,COUNT(lat) as count from webserver GROUP by lat,lon"
)
for e in db.cur.fetchall():
radius = e[3] * 5
folium.CircleMarker(
location=[e[0], e[1]],
popup="{} ({})".format(e[2], e[3]),
radius=15 if radius > 15 else radius).add_to(folium_map)
folium_map.save('visualisation/coords.html')
def get_events():
try:
date = request.args.get('date')
except AttributeError:
return failure_response('Usage: /get_events?date=<YYYY-MM-DD>')
start_timestamp = datetime.strptime(date, '%Y-%m-%d')
end_timestamp = start_timestamp + timedelta(days=1)
query = '''
SELECT *
FROM service_notifications
WHERE timestamp BETWEEN '{}' and '{}'
'''.format(start_timestamp, end_timestamp)
return db.exec_query(query)
def poll_tg_bot():
while True:
bot = Bot()
query = """
SELECT id, level, service, host(host), text, to_char(timestamp, 'HH24:MI:SS\tDD Mon') as time
FROM service_notifications
WHERE processed=false
"""
result = db.exec_query(query)
for line in result:
tmp_line = {i: line[i] for i in line if i != 'id'}
bot.send_msg(tmp_line)
update_query = '''
UPDATE service_notifications
SET processed=true
WHERE id={}
'''.format(line['id'])
db.exec_query(update_query)
sleep(60)
def process_domain(domain: PreparedDomain, category: str, reevaluate: bool = False, restoreconn: bool = False,
cz_omit: bool = True, subdom_omit: bool = False,
output: object = None, db_conn: object = None) -> None:
"""
It will process given domain. The base evaluation function.
:param domain: Domain name
:param category: Category for this page
:param reevaluate: Reevaluate if found in db
:param restoreconn: It will call external script in case no WHOIS connectivity
:param cz_omit: It will omit domain within .cz domain zone
:param subdom_omit: It will omit domain on 3rd and higher level
:param output: Reference to opened file to write output log
:param db_conn: Reference to opened database connection
:return:
"""
if db_conn is None:
db.open_connection()
db_conn_created = True
else:
db.take_connection(db_conn)
db_conn_created = False
print("\n========================================================================\nProcessing page:", domain)
di = page_evaluator.get_domain_info(domain.domain)
if not di.get('serverIp'):
print("[ERROR][NO_DNS]", domain, 'Unable to translate domain. Skipping...')
return
if cz_omit:
if di.get('tld') != 'cz':
print("[NOTICE][NO_CZ]", domain, "omitted, not in CZ zone.")
return
if subdom_omit:
if di.get('subdomain').lower() not in ['', 'www', None]:
print("[NOTICE][SUBDOM]", domain, "is subdomain, omitted.")
return
if is_domain_evaluated(di.get('baseDomain')):
if reevaluate:
db.exec_query(delete_result.format(domain=di.get('baseDomain')))
db.commit()
else:
print("[NOTICE] Been evaluated in past:", domain, "skipping...")
return
whois_info = whois_parser.probe_domain(di['baseDomain'], restoreconn)
if whois_info is {}:
print("[ERROR][WHOIS_NIC] - no valid response for ", domain)
return
# Get WHOIS and GEO data about IP
try:
whois_ip_info = whois_parser.probe_ip(di['serverIp'])
except Exception as e:
print("[ERROR][WHOIS_IP] for ", di['serverIp'], e)
return
server_stats = wserver.probe_server(di['baseDomain'])
server_stats_https = wserver.probe_server(di['baseDomain'], True)
https_supported = check_https(server_stats_https)
if whois_info.get('holderId'):
contact_info = whois_parser.probe_contact(whois_info.get('holderId'), restoreconn)
else:
contact_info = {}
evaluated = page_evaluator.evaluate_domain(di, whois_info, whois_ip_info)
# Print given note
if domain.note:
print("[DOMAIN-NOTE]", domain.note)
print("[INFO][DOMAIN] b.dom: {bDomain}, s.IP: {IP}, s.IPv6 {IP6}, s.IP name: {IP_name}".format(
bDomain=di.get('baseDomain'),
IP=di.get('serverIp'),
IP6=di.get('serverIp6'),
IP_name=di.get('netDomainFull')))
if whois_info is not {}:
print("[INFO][CZNIC_WHOIS] holder: {hold}, NS: {nsset}".format(hold=whois_info.get('holderName'),
nsset=whois_info.get('nsset')))
if contact_info is not {}:
print("[INFO][CZNIC_CONTACT] email: {email}, ident: {ident}".format(email=contact_info.get('email'),
ident=contact_info.get('ident')))
print("[INFO][NET] email: {email}, org: {ident}, GEO {geo}".format(email=whois_ip_info.contact_email,
ident=whois_ip_info.get_organization(),
geo=whois_ip_info.geo_ip))
print("[INFO][WEBSRV] stat: {status}, sign: {sign}, HTTPS: {https_sup}".format(status=server_stats.get('status'),
sign=server_stats.get('server'),
https_sup=https_supported))
print("\n[EVAL][RESULTS] ", evaluated)
# Write to DB if was given category to store results
if category:
db.exec_query(add_result.format(
domain=di.get('baseDomain'),
res_rev=evaluated.method_reverse_dns,
res_known=evaluated.method_known_hoster,
res_whois=evaluated.method_whois,
res_email=evaluated.method_email,
s_server=server_stats.get('server'),
s_powerer=server_stats.get('x_powerer'),
s_type=server_stats.get('c_type'),
s_ip=di.get('serverIp'),
s_ip_name=di.get('netDomainFull'),
s_ip_domain=di.get('netDomain'),
dm_keyset=whois_info.get('keyset'),
dm_holdername=whois_info.get('holderName'),
ip_holdername=whois_ip_info.get_organization(),
category=category,
original_url=domain.original_url,
note=domain.note,
ipv6_addr=di.get('serverIp6'),
https_support=https_supported
))
db.commit()
if output is not None or False:
output.write("{dom}\t{m_wh}\t{m_rd}\t{m_h}\t{m_e}\t{s_ip}\t{d_hold}\t{ip_org}\t{ip_em}\n".format(
dom=di.get('domain'),
m_wh=EvaluatedPage.bool2switcher(evaluated.method_whois),
m_rd=EvaluatedPage.bool2switcher(evaluated.method_reverse_dns),
m_h=EvaluatedPage.bool2switcher(evaluated.method_known_hoster),
m_e=EvaluatedPage.bool2switcher(evaluated.method_email),
s_ip=di.get('serverIp'),
d_hold=whois_info.get('holderName'),
ip_org=whois_ip_info.get_organization(),
ip_em=whois_ip_info.contact_email
))
if db_conn_created is True:
db.close_connection()
SELECT id, level, service, host(host), text, to_char(timestamp, 'HH24:MI:SS\tDD Mon') as time
FROM service_notifications
WHERE processed=false
"""
result = db.exec_query(query)
for line in result:
tmp_line = {i: line[i] for i in line if i != 'id'}
bot.send_msg(tmp_line)
update_query = '''
UPDATE service_notifications
SET processed=true
WHERE id={}
'''.format(line['id'])
db.exec_query(update_query)
sleep(60)
if __name__ == '__main__':
bot = Bot()
# bot.test()
query = """
SELECT level, service, host(host), text, to_char(timestamp, 'HH24:MI:SS\tDD Mon') as time
FROM service_notifications
"""
msg = db.exec_query(query)
bot.send_msg(msg['data'][0])