def run(self):
database = self.sourcedb
connt = sqlite3.connect(database)
#create_database("result.sqlite")
result = self.desdb
connt2 = sqlite3.connect(result)
cur = connt.cursor()
rows = cur.execute("SELECT url FROM http_requests")
# rows = cur.execute("SELECT url FROM http_requests WHERE id BETWEEN "+str(startid)+" AND "+str(stopid)+";")
connt2.cursor().execute("""CREATE TABLE IF NOT EXISTS unsafe_url
(URL TEXT NOT NULL,
RESULT TEXT NOT NULL);""")
for row in rows:
url = row[0]
print(url)
apikey = 'AIzaSyArUsQxoVRRMQ4YhDR9ae51gWmuXsP5sw4'
sb = safebrowsing.LookupAPI(apikey)
resp = sb.threat_matches_find(url)
if "matches" in resp:
connt2.cursor().execute(
"""INSERT INTO unsafe_url
(URL, RESULT)
values (?,?)""", (str(url), str(resp)))
connt2.commit()
def sendlink(link):
import safebrowsing
import json
apikey = myconf.get('lookup.key')
sb = safebrowsing.LookupAPI(apikey)
resp = sb.threat_matches_find(link)
if u'matches' in resp:
return "unsafe"
return "ok"
def start(conn, conn2, startid, stopid):
cur = conn.cursor()
rows = cur.execute("SELECT url FROM http_requests")
# rows = cur.execute("SELECT url FROM http_requests WHERE id BETWEEN "+str(startid)+" AND "+str(stopid)+";")
conn2.cursor().execute("""CREATE TABLE IF NOT EXISTS unsafe_url
(URL TEXT NOT NULL,
RESULT TEXT NOT NULL);""")
for row in rows:
url = row[0]
print(url)
apikey = 'AIzaSyArUsQxoVRRMQ4YhDR9ae51gWmuXsP5sw4'
sb = safebrowsing.LookupAPI(apikey)
resp = sb.threat_matches_find(url)
if "matches" in resp:
conn2.cursor().execute(
"""INSERT INTO unsafe_url
(URL, RESULT)
values (?,?)""", (str(url), str(resp)))
conn2.commit()
try:
return datetime.strptime(create_time, FMT) - datetime.strptime(post_time, FMT)
except:
return -1
def time_after_profile_creation(df):
df['TIME_AFTER_PFP_CREATION'] = df.apply(
lambda x: get_time(x["CREATEDAT"], x["OPEN_DATE"]), axis=1)
print('-------Time after Profile Creation--------')
print(df.head()[['CONTENT', "TIME_AFTER_PFP_CREATION"]])
apikey = 'AIzaSyAYeCUJwGYBKRdvifnR3ggtuR12t0xe3vA'
sb = safebrowsing.LookupAPI(apikey)
def is_phising_links(links):
for link in links:
resp = sb.threat_matches_find(link)
if "matches" in resp and (len(resp["matches"]) > 0):
return True
return False
def is_phising_site(df):
df['CONTAINS_PHISHING'] = df["URL_LIST"].apply(
lambda x: is_phising_links(x))
print('-------Contains Phishing Sites--------')
def scan_hostname(hostname, SerialNumber, lines, Proxy, conn, site_infos):
"""
try scan a hostname and get informations back
(HTTP code, page title, IP address, ASN, abuse email etc).
:param hostname: the hostname present in the certificate
:param SerialNumber: the serial number of the certificate
:param lines: list of user-agents strings
:param Proxy: proxy settings
:param conn: database connection
:param site_infos: informations extracted on the net for the given hostname
:return: True if everything went fine, False if any problem has been encountered
"""
title = ""
try:
r = get_requests(hostname, lines, conn, Proxy)
if r is not None:
hues.success('HTTP ' + str(r.status_code) + ' - ' + hostname)
# retrieve the title of the homepage
title = get_webpage_title(r)
# retrieve ASN informations
ipaddr = socket.gethostbyname(hostname)
asn, asn_cidr, asn_country_code, asn_description, asn_abuse_email = get_ASN_Infos(
ipaddr)
# retrieve Google Safe Browsing Lookup API status for this hostname
if Safe_Browsing_API_Key is not '':
sb = safebrowsing.LookupAPI(Safe_Browsing_API_Key)
safe_browsing_status = sb.threat_matches_find(hostname)
else:
safe_browsing_status = "No API key in config file"
# build the content of the alert file using certificate / webpage / ASN informations
site_infos = {
'hostname': hostname,
'http_code': r.status_code,
'cert_serial_number': SerialNumber,
'webpage_title': title,
'ip_addr': ipaddr,
'asn': asn,
'asn_cidr': asn_cidr,
'asn_country_code': asn_country_code,
'asn_description': asn_description,
'asn_abuse_email': asn_abuse_email,
'safe_browsing_status': safe_browsing_status
}
return site_infos
else:
return {}
except KeyboardInterrupt:
print("scan_hostname() - Interrupt received, stopping ...")
print("start - committing, closing DB")
conn.commit
conn.close
print("ending - committing, closing DB")
sys.exit(0)
except Exception as ex:
hues.error("scan_hostname() - any other kind of error: {}".format(ex))
return {}