def TryDLPK(TABLEname, InvTABLEname, DLDir, SQL, PROXY, LOG, UAFILE):
from tools.download import TryPKDownload
# Search in main Table for StillTryDownload column
rows = SQL.SQLiteSearchNotDownloaded(TABLEname)
try:
for row in rows:
siteDomain = row[1]
IPaddress = row[2]
if IPaddress:
rASN = NetInfo()
if rASN.GetASN(IPaddress):
ASN = rASN.GetASN(IPaddress).strip('\"')
else:
ASN = None
else:
ASN = None
if row[0].startswith('https'):
siteURL = row[0]
if row[0].startswith('http'):
siteURL = str(row[0])
else:
siteURL = 'http://' + row[0]
TryPKDownload(siteURL, siteDomain, IPaddress, TABLEname,
InvTABLEname, DLDir, SQL, PROXY, LOG, UAFILE, ASN)
except:
err = sys.exc_info()
LOG.error("TryDLPK module error: " + str(err))
def SiteURLSQL(item, LOG, SQL, TABLEname, PROXY, UAFILE, UAG):
# remove URL containing UID-style strings
siteURL = quote(
re.split("(?:[0-9a-fA-F]:?){32}", item['page']['url'])[0], ':/')
dn = dirname(siteURL)
# Test if entry still exist in DB
if SQL.SQLiteVerifyEntry(TABLEname, dn) == 0:
now = str(TimestampNow().Timestamp())
siteDomain = urlparse(item['page']['url']).netloc
source_url = item['result'].replace("/api/v1", "")
try:
IPaddress = socket.gethostbyname(siteDomain)
if IPaddress:
rASN = NetInfo()
ASN = rASN.GetASN(IPaddress).strip('\"')
else:
pass
# can't resolv
except:
IPaddress = ""
ASN = ""
# HTTP connection
try:
proxies = {'http': PROXY, 'https': PROXY}
UA = UAG.ChooseUA(UAFILE)
user_agent = {'User-agent': UA}
try:
r = requests.get(siteURL,
headers=user_agent,
proxies=proxies,
allow_redirects=True)
lastHTTPcode = str(r.status_code)
except ValueError:
# No user-agent configured
r = requests.get(siteURL,
proxies=proxies,
allow_redirects=True)
lastHTTPcode = str(r.status_code)
except requests.exceptions.Timeout:
lastHTTPcode = "timeout"
except requests.exceptions.ConnectionError:
lastHTTPcode = "aborted"
except:
lastHTTPcode = "---"
pass
except Exception as e:
# Unknown status code
LOG.error("Connection error: {}".format(e))
pass
LOG.info(siteURL + " " + siteDomain + " " + IPaddress + " " +
source_url + " " + now + " " + lastHTTPcode)
SQL.SQLiteInsertPK(TABLEname, siteURL, siteDomain, IPaddress,
source_url, now, lastHTTPcode, ASN)
else:
LOG.debug("Entry still known: " + siteURL)
pass
def AddUniqueURL(URLadd, LOG, SQL, TABLEname, PROXY, UAFILE):
UAG = UAgent()
# add schema
if URLadd.startswith("http://") or URLadd.startswith("https://"):
pass
else:
URLadd = "http://{}".format(URLadd)
# remove URL containing UID-style strings
siteURL = re.split("(?:[0-9a-fA-F]:?){32}", URLadd.rstrip())[0]
# Test if entry still exist in DB
if SQL.SQLiteVerifyEntry(TABLEname, siteURL) == 0:
now = str(TimestampNow().Timestamp())
siteDomain = urlparse(URLadd).netloc
source_url = "Manual"
try:
IPaddress = socket.gethostbyname(siteDomain)
rASN = NetInfo()
ASN = rASN.GetASN(IPaddress).strip('\"')
# can't resolv
except:
IPaddress = ""
ASN = ""
# HTTP connection
try:
proxies = {'http': PROXY, 'https': PROXY}
UA = UAG.ChooseUA(UAFILE)
user_agent = {'User-agent': UA}
try:
r = requests.get(siteURL,
headers=user_agent,
proxies=proxies,
allow_redirects=True,
timeout=(5, 12))
lastHTTPcode = str(r.status_code)
except ValueError:
# No user-agent configured
r = requests.get(siteURL,
proxies=proxies,
allow_redirects=True,
timeout=(5, 12))
lastHTTPcode = str(r.status_code)
except requests.exceptions.Timeout:
lastHTTPcode = "timeout"
except requests.exceptions.ConnectionError:
lastHTTPcode = "aborted"
except:
lastHTTPcode = "---"
err = sys.exc_info()
LOG.error("HTTP error: " + str(err))
pass
except:
# Unknown status code
err = sys.exc_info()
LOG.error("Connection error: " + str(err))
pass
# Add data into database
LOG.info(siteURL)
SQL.SQLiteInsertPK(TABLEname, siteURL, siteDomain, IPaddress,
source_url, now, lastHTTPcode, ASN)
else:
LOG.info("Entry still known: " + siteURL)
pass
def SiteURLSQL(phishtank_file, entry, LOG, SQL, TABLEname, PROXY, UAFILE, UAG):
# remove URL containing UID-style strings
siteURL = quote(re.split("(?:[0-9a-fA-F]:?){32}", entry['url'])[0], ':/')
dn = dirname(siteURL)
# Test if entry still exist in DB
if SQL.SQLiteVerifyEntry(TABLEname, dn) is 0:
IPaddress = entry['details'][0]['ip_address']
source_url = entry['phish_detail_url']
siteDomain = urlparse(entry['url']).netloc
now = str(TimestampNow().Timestamp())
try:
IPaddress = socket.gethostbyname(siteDomain)
if IPaddress:
rASN = NetInfo()
ASN = rASN.GetASN(IPaddress).strip('\"')
else:
pass
# can't resolv
except:
IPaddress = ""
ASN = ""
# HTTP connection
try:
proxies = {'http': PROXY, 'https': PROXY}
UA = UAG.ChooseUA(UAFILE)
user_agent = {'User-agent': UA}
try:
r = requests.get(siteURL,
headers=user_agent,
proxies=proxies,
allow_redirects=True,
timeout=(5, 12))
# Follow redirect and add new URI to database
if (len(r.history) > 1) and ("301" in str(
r.history[-1])) and (siteURL != r.url) and (
siteURL.split('/')[:-1] != r.url.split('/')[:-2]
) and (siteURL + '/' != r.url):
lastHTTPcode = str(r.status_code)
SQL.SQLiteInsertPK(TABLEname, r.url, siteDomain, IPaddress,
source_url, now, lastHTTPcode, ASN)
else:
pass
lastHTTPcode = str(r.status_code)
except ValueError:
# No user-agent configured
r = requests.get(siteURL,
proxies=proxies,
allow_redirects=True,
timeout=(5, 12))
lastHTTPcode = str(r.status_code)
except requests.exceptions.Timeout:
lastHTTPcode = "timeout"
except requests.exceptions.ConnectionError:
lastHTTPcode = "aborted"
except:
lastHTTPcode = "---"
err = sys.exc_info()
LOG.error("HTTP error: " + str(err))
pass
except:
# Unknown status code
err = sys.exc_info()
LOG.error("Connection error: " + str(err))
pass
# Add data into database
LOG.info(siteURL)
SQL.SQLiteInsertPK(TABLEname, siteURL, siteDomain, IPaddress,
source_url, now, lastHTTPcode, ASN)
else:
LOG.debug("Entry still known: " + siteURL)
pass
def SiteURLSQL(SearchString, line, LOG, SQL, TABLEname, PROXY, UAFILE, UAG):
# remove URL containing UID-style strings
siteURL = quote(re.split("(?:[0-9a-fA-F]:?){32}", line[0])[0], ':/')
if siteURL.startswith('https:'):
siteDomain = siteURL.split('/')[2]
else:
siteDomain = siteURL.split('/')[0]
siteURL = "http://" + siteURL
dn = dirname(siteURL)
# Test if entry still exist in DB
if SQL.SQLiteVerifyEntry(TABLEname, dn) is 0:
# Proceed to informations retrieve
now = str(TimestampNow().Timestamp())
source_url = "https://urlquery.net/" + line[1]
try:
IPaddress = socket.gethostbyname(siteDomain)
if IPaddress:
rASN = NetInfo()
ASN = rASN.GetASN(IPaddress).strip('\"')
else:
pass
# can't resolv
except:
IPaddress = ""
ASN = ""
# HTTP connection
try:
proxies = {'http': PROXY, 'https': PROXY}
UA = UAG.ChooseUA(UAFILE)
user_agent = {'User-agent': UA}
try:
r = requests.get(siteURL,
headers=user_agent,
proxies=proxies,
allow_redirects=True)
# Follow redirect and add new URI to database
if (len(r.history) > 1) and ("301" in str(
r.history[-1])) and (siteURL != r.url) and (
siteURL.split('/')[:-1] != r.url.split('/')[:-2]
) and (siteURL + '/' != r.url):
lastHTTPcode = str(r.status_code)
SQL.SQLiteInsertPK(TABLEname, r.url, siteDomain, IPaddress,
source_url, now, lastHTTPcode, ASN)
else:
pass
lastHTTPcode = str(r.status_code)
except ValueError:
# No user-agent configured
r = requests.get(siteURL,
proxies=proxies,
allow_redirects=True)
lastHTTPcode = str(r.status_code)
except requests.exceptions.Timeout:
lastHTTPcode = "timeout"
except requests.exceptions.ConnectionError:
lastHTTPcode = "aborted"
except:
lastHTTPcode = "---"
pass
except Exception as e:
# Unknown status code
LOG.error("Connection error: {}".format(e))
pass
# Add data into database
LOG.info(siteURL + " " + siteDomain + " " + IPaddress + " " +
source_url + " " + now + " " + lastHTTPcode)
SQL.SQLiteInsertPK(TABLEname, siteURL, siteDomain, IPaddress,
source_url, now, lastHTTPcode, ASN)
else:
LOG.debug("Entry still known: " + siteURL)
pass