def reportMajorError(file, description):
from send_email import sendAdminAlert
script_file_name = os.path.basename(__file__)
sendAdminAlert("Problem with script: (major error) "+file,
"There's been an issue...\n\nProblem with script: "+file+"\nDescription of problem: "+description+"\n\nDetected by "+script_file_name)
email_sent = 1
# report error in db:
sql = "INSERT INTO error_reports_5(file, description, email_sent) VALUES(%s, %s, %s)"
cur.execute(sql, (file, description, email_sent))
con.commit()
def tweepyError(e, location, search_term=None):
print e
if search_term:
print "search term:", search_term
if "code" in e.message[0]:
error_code = e.message[0]['code']
if isinstance(error_code, (int, long)):
if error_code == 88:
print time.strftime(
"%c"
) + " Rate limit (in " + location + ")\nWait 90 secs..."
time.sleep(90)
elif e.message[0]['code'] == 130:
print time.strftime(
"%c"
) + " Twitter over capacity (in " + location + "). Wait 90 secs..."
time.sleep(90)
else:
from send_email import sendAdminAlert
import os
script_file_name = os.path.basename(__file__)
error_message = "Unrecognised (as in not set in conditional) error code in functon tweepyError(). Error code: " + str(
error_code)
sendAdminAlert(
"Error in " + script_file_name,
"Python script: " + script_file_name +
"\nError reported: " + str(error_message))
exit()
else:
from send_email import sendAdminAlert
import os
script_file_name = os.path.basename(__file__)
error_message = 'Conditional fail: isinstance(error_code, ( int, long ) )'
sendAdminAlert(
"Error in " + script_file_name, "Python script: " +
script_file_name + "\nError reported: " + str(error_message))
exit()
def reportError(file, description):
email_sent = 0
# check when last email was sent:
cur.execute("SELECT COUNT(*) as num from error_reports_5 WHERE file = %s AND date_reported > DATE_SUB(NOW(), INTERVAL 6 HOUR)", (file,))
rows = cur.fetchall()
result = 0;
for row in rows:
result = int(row[0]);
# send email as error reported more than 6 hours ago:
if result == 0:
from send_email import sendAdminAlert
script_file_name = os.path.basename(__file__)
sendAdminAlert("Problem with script: "+file,
"There's been an issue...\n\nProblem with script: "+file+"\nDescription of problem: "+description+"\n\nDetected by "+script_file_name)
email_sent = 1
# report error in db:
sql = "INSERT INTO error_reports_5(file, description, email_sent) VALUES(%s, %s, %s)"
cur.execute(sql, (file, description, email_sent))
con.commit()
sql = "INSERT INTO gsb_update_log_5(num_urls_gglsbl4, num_unique_urls_gglsbl4) VALUES(%s, %s)"
cur.execute(sql, (len(all_rows), len(gsb_url_hash_prefix_dict)))
print "len: {:,}".format(len(all_rows))
except:
print "We have an error:", sys.exc_info()[1]
import os
import sys
import traceback
from send_email import sendAdminAlert
script_file_name = os.path.basename(__file__)
error_string = ""
for frame in traceback.extract_tb(sys.exc_info()[2]):
fname,lineno,fn,text = frame
error_string += "\nError in %s on line %d" % (fname, lineno)
sendAdminAlert("Error in "+script_file_name,
"Python script: "+script_file_name+"\nError reprted: "+ str(sys.exc_info()[1])+"\nLine:" + str(error_string))
print "waiting 90 seconds"
time.sleep(90)
end = time.time()
print "time taken to update GSB: {0:.0f}".format(end-start)+"secs ({0:.0f}".format((end-start)/60)+"mins)"
mainEnd = time.time()
#print "time taken to complete whole script: {0:.0f}".format(mainEnd-mainStart)+"secs ({0:.0f}".format((mainEnd-mainStart)/60)+"mins)"
runTime = (mainEnd-mainStart)
#seconds_to_run = 1800 #30 minutes
#seconds_to_run = 600 #10 minutes
#seconds_to_run = 900 #15 minutes # tried this on Wed 4 Oct 2017 and just took 30 minute to update
seconds_to_run = 300 #5 minutes, ready to put live after adjusting discard_fair_use_policy to true
# 28 Mar 2018 13:33: tried setting 5 minutes intervals but script took 27 minutes to complete
def URLLookup_v2(domain_hashes, gsb_urls):
# twitter_urls = (hash_list)
# twitter_urls_dict = (tweet_ids)
phish_count = 0
malware_count = 0
phishing_matches = {}
counter = 0
hash_prefix_counter = 0
printCounter = 0
num_urls = len(domain_hashes)
malware_domains = []
phishing_domains = []
for url in domain_hashes:
#print url
url_hashes = domain_hashes[url]
for url_hash in url_hashes:
#print url_hash
hash_prefix = sqlite3.Binary(url_hash[0:4])
hash_prefix = str(hash_prefix).encode('hex')
if hash_prefix in gsb_urls:
if (lookup_gsb_full_hash(sqlite3.Binary(url_hash))):
hash_prefix_counter += 1
database_lock = True
while database_lock:
try:
print "url hash prefix match!"
#gsblookup = sbl.lookup_hash(url_hash)
gsblookup = sbl.lookup_url(url)
if gsblookup:
cur = con.cursor()
sql = "INSERT INTO gsb_full_hash_log_certstream_5(url, hash_prefix, full_hash) VALUES(%s, %s, %s)"
cur.execute(
sql,
(url[0:500], sqlite3.Binary(url_hash[0:4]),
sqlite3.Binary(url_hash)))
print "url full hash match!"
for i in gsblookup:
print i
if str(i) == "MALWARE/ANY_PLATFORM/URL":
cur.execute(
"UPDATE gsb_full_hash_log_certstream_5 SET malware = '1' WHERE full_hash = %s",
(url_hash, ))
con.commit()
print "malware"
malware_count += 1
#sys.stdout.write('\r-')
#sys.stdout.flush()
#malware_domains.append()
if str(
i
) == "SOCIAL_ENGINEERING/ANY_PLATFORM/URL":
cur.execute(
"UPDATE gsb_full_hash_log_certstream_5 SET social_engineering = '1' WHERE full_hash = %s",
(url_hash, ))
con.commit()
print "phishing " #,url_data[0]
phish_count += 1
#logPhishingURL(url, 3, twitter_urls_dict[url], redirection_chain_url_lookup)
#markTweetsPhishy(url, twitter_urls_dict[url])
phishing_matches[url] = (url_hash)
else: # url hash prefix match but full hash not in GSB i.e. different URL. Mark in db so doesn't get checked again
cur = con.cursor()
sql = "INSERT INTO gsb_full_hash_log_certstream_5(hash_prefix, full_hash, not_in_gsb) VALUES(%s, %s, %s)"
cur.execute(sql,
(sqlite3.Binary(url_hash[0:4]),
sqlite3.Binary(url_hash), 1))
database_lock = False
except (RuntimeError, IntegrityError,
urllib2.HTTPError, urllib2.URLError,
SocketError) as e:
print e
print url
log(
"tpl_v3.2-output.txt", "error: " +
str(e.message) + "\nURL: " + url + "\n")
print "waiting 5 seconds..."
time.sleep(5)
database_lock = False
except sqlite3.OperationalError:
print("database locked, waiting 5 seconds...")
log("tpl_v3.2-output.txt",
"gglsbl3 database is locked")
time.sleep(5)
database_lock = False
except KeyError:
# 18 Jun 2018, disabling error: 'exceptions.KeyError'>, KeyError('matches',)
# as appearing every minute or so
print "Looks like a key error:", sys.exc_info()[1]
log(
"certstream-url-checker-v2-phishing_5.txt",
"Looks like a key error: " +
str(sys.exc_info()[1]))
print "URL:", url
time.sleep(5)
database_lock = False
except:
print "We have an error:", sys.exc_info()[1]
print sys.exc_info()
import os
import traceback
from send_email import sendAdminAlert
script_file_name = os.path.basename(__file__)
error_string = ""
for frame in traceback.extract_tb(
sys.exc_info()[2]):
fname, lineno, fn, text = frame
error_string += "\nError in %s on line %d" % (
fname, lineno)
print error_string
sendAdminAlert(
"Error (within URLLookup_v2 loop) in " +
script_file_name,
"Python script: " + script_file_name +
"\nError reprted: " + str(sys.exc_info()[1]) +
"\nLine:" + str(error_string))
print "waiting 90 seconds"
time.sleep(90)
database_lock = False
else:
print "URL already in gsb_full_hash_lookup"
#j+=1
counter += 1
printCounter += 1
if (printCounter == 10):
#progress_bar(counter, num_urls, '%s of %s (hash_prefix_counter: %s)' % (counter, num_urls, hash_prefix_counter))
progress_bar(counter, num_urls, '%s of %s' % (counter, num_urls))
printCounter = 0
return (phish_count, malware_count, phishing_matches)
time.sleep(30)
except mdb.Error, e:
print "We have an error:", sys.exc_info()[1]
import os
import sys
import traceback
from send_email import sendAdminAlert
script_file_name = os.path.basename(__file__)
error_string = ""
for frame in traceback.extract_tb(sys.exc_info()[2]):
fname, lineno, fn, text = frame
error_string += "\nError in %s on line %d" % (fname, lineno)
print error_string
sendAdminAlert(
"MySQL error in twitter stream sample",
"Python script: " + script_file_name + "\nError reported: " +
str(sys.exc_info()[1]) + "\nLine:" + str(error_string) +
"\n\n" + (str(getTS())))
print "Will try to reconnect to MySQL server in 90 seconds..."
print getTS()
time.sleep(90)
#con = mdb.connect(config['mysql']['host'], config['mysql']['username'], config['mysql']['password'], config['mysql']['database_5'],charset='utf8',cursorclass = mdb.cursors.SSCursor)
#cursor = con.cursor()
#con.autocommit(True)
except:
print "Unknown error...", sys.exc_info()[0]
print getTS()
cur = con.cursor()
sql = "INSERT INTO twitter_stream_error_log(twitter_sample_stream, error_reported, notes) VALUES(%s, %s, %s)"
print getTS()
time.sleep(90)
except mdb.Error, e:
print "We have an error:", sys.exc_info()[1]
import os
import sys
import traceback
from send_email import sendAdminAlert
script_file_name = os.path.basename(__file__)
error_string = ""
for frame in traceback.extract_tb(sys.exc_info()[2]):
fname, lineno, fn, text = frame
error_string += "\nError in %s on line %d" % (fname, lineno)
print error_string
sendAdminAlert(
"MySQL error in main twitter stream (i.e URL filter)",
"Python script: " + script_file_name + "\nError reported: " +
str(sys.exc_info()[1]) + "\nLine:" + str(error_string))
print "Will try to reconnect to MySQL server in 90 seconds..."
print getTS()
time.sleep(90)
#con = mdb.connect(config['mysql']['host'], config['mysql']['username'], config['mysql']['password'], config['mysql']['database_5'],charset='utf8',cursorclass = mdb.cursors.SSCursor)
#cursor = con.cursor()
#con.autocommit(True)
except:
print "We have an error:", sys.exc_info()[1]
import os
import sys
import traceback
from send_email import sendAdminAlert