def reportMajorError(file, description): from send_email import sendAdminAlert script_file_name = os.path.basename(__file__) sendAdminAlert("Problem with script: (major error) "+file, "There's been an issue...\n\nProblem with script: "+file+"\nDescription of problem: "+description+"\n\nDetected by "+script_file_name) email_sent = 1 # report error in db: sql = "INSERT INTO error_reports_5(file, description, email_sent) VALUES(%s, %s, %s)" cur.execute(sql, (file, description, email_sent)) con.commit()
def tweepyError(e, location, search_term=None): print e if search_term: print "search term:", search_term if "code" in e.message[0]: error_code = e.message[0]['code'] if isinstance(error_code, (int, long)): if error_code == 88: print time.strftime( "%c" ) + " Rate limit (in " + location + ")\nWait 90 secs..." time.sleep(90) elif e.message[0]['code'] == 130: print time.strftime( "%c" ) + " Twitter over capacity (in " + location + "). Wait 90 secs..." time.sleep(90) else: from send_email import sendAdminAlert import os script_file_name = os.path.basename(__file__) error_message = "Unrecognised (as in not set in conditional) error code in functon tweepyError(). Error code: " + str( error_code) sendAdminAlert( "Error in " + script_file_name, "Python script: " + script_file_name + "\nError reported: " + str(error_message)) exit() else: from send_email import sendAdminAlert import os script_file_name = os.path.basename(__file__) error_message = 'Conditional fail: isinstance(error_code, ( int, long ) )' sendAdminAlert( "Error in " + script_file_name, "Python script: " + script_file_name + "\nError reported: " + str(error_message)) exit()
def reportError(file, description): email_sent = 0 # check when last email was sent: cur.execute("SELECT COUNT(*) as num from error_reports_5 WHERE file = %s AND date_reported > DATE_SUB(NOW(), INTERVAL 6 HOUR)", (file,)) rows = cur.fetchall() result = 0; for row in rows: result = int(row[0]); # send email as error reported more than 6 hours ago: if result == 0: from send_email import sendAdminAlert script_file_name = os.path.basename(__file__) sendAdminAlert("Problem with script: "+file, "There's been an issue...\n\nProblem with script: "+file+"\nDescription of problem: "+description+"\n\nDetected by "+script_file_name) email_sent = 1 # report error in db: sql = "INSERT INTO error_reports_5(file, description, email_sent) VALUES(%s, %s, %s)" cur.execute(sql, (file, description, email_sent)) con.commit()
sql = "INSERT INTO gsb_update_log_5(num_urls_gglsbl4, num_unique_urls_gglsbl4) VALUES(%s, %s)" cur.execute(sql, (len(all_rows), len(gsb_url_hash_prefix_dict))) print "len: {:,}".format(len(all_rows)) except: print "We have an error:", sys.exc_info()[1] import os import sys import traceback from send_email import sendAdminAlert script_file_name = os.path.basename(__file__) error_string = "" for frame in traceback.extract_tb(sys.exc_info()[2]): fname,lineno,fn,text = frame error_string += "\nError in %s on line %d" % (fname, lineno) sendAdminAlert("Error in "+script_file_name, "Python script: "+script_file_name+"\nError reprted: "+ str(sys.exc_info()[1])+"\nLine:" + str(error_string)) print "waiting 90 seconds" time.sleep(90) end = time.time() print "time taken to update GSB: {0:.0f}".format(end-start)+"secs ({0:.0f}".format((end-start)/60)+"mins)" mainEnd = time.time() #print "time taken to complete whole script: {0:.0f}".format(mainEnd-mainStart)+"secs ({0:.0f}".format((mainEnd-mainStart)/60)+"mins)" runTime = (mainEnd-mainStart) #seconds_to_run = 1800 #30 minutes #seconds_to_run = 600 #10 minutes #seconds_to_run = 900 #15 minutes # tried this on Wed 4 Oct 2017 and just took 30 minute to update seconds_to_run = 300 #5 minutes, ready to put live after adjusting discard_fair_use_policy to true # 28 Mar 2018 13:33: tried setting 5 minutes intervals but script took 27 minutes to complete
def URLLookup_v2(domain_hashes, gsb_urls): # twitter_urls = (hash_list) # twitter_urls_dict = (tweet_ids) phish_count = 0 malware_count = 0 phishing_matches = {} counter = 0 hash_prefix_counter = 0 printCounter = 0 num_urls = len(domain_hashes) malware_domains = [] phishing_domains = [] for url in domain_hashes: #print url url_hashes = domain_hashes[url] for url_hash in url_hashes: #print url_hash hash_prefix = sqlite3.Binary(url_hash[0:4]) hash_prefix = str(hash_prefix).encode('hex') if hash_prefix in gsb_urls: if (lookup_gsb_full_hash(sqlite3.Binary(url_hash))): hash_prefix_counter += 1 database_lock = True while database_lock: try: print "url hash prefix match!" #gsblookup = sbl.lookup_hash(url_hash) gsblookup = sbl.lookup_url(url) if gsblookup: cur = con.cursor() sql = "INSERT INTO gsb_full_hash_log_certstream_5(url, hash_prefix, full_hash) VALUES(%s, %s, %s)" cur.execute( sql, (url[0:500], sqlite3.Binary(url_hash[0:4]), sqlite3.Binary(url_hash))) print "url full hash match!" for i in gsblookup: print i if str(i) == "MALWARE/ANY_PLATFORM/URL": cur.execute( "UPDATE gsb_full_hash_log_certstream_5 SET malware = '1' WHERE full_hash = %s", (url_hash, )) con.commit() print "malware" malware_count += 1 #sys.stdout.write('\r-') #sys.stdout.flush() #malware_domains.append() if str( i ) == "SOCIAL_ENGINEERING/ANY_PLATFORM/URL": cur.execute( "UPDATE gsb_full_hash_log_certstream_5 SET social_engineering = '1' WHERE full_hash = %s", (url_hash, )) con.commit() print "phishing " #,url_data[0] phish_count += 1 #logPhishingURL(url, 3, twitter_urls_dict[url], redirection_chain_url_lookup) #markTweetsPhishy(url, twitter_urls_dict[url]) phishing_matches[url] = (url_hash) else: # url hash prefix match but full hash not in GSB i.e. different URL. Mark in db so doesn't get checked again cur = con.cursor() sql = "INSERT INTO gsb_full_hash_log_certstream_5(hash_prefix, full_hash, not_in_gsb) VALUES(%s, %s, %s)" cur.execute(sql, (sqlite3.Binary(url_hash[0:4]), sqlite3.Binary(url_hash), 1)) database_lock = False except (RuntimeError, IntegrityError, urllib2.HTTPError, urllib2.URLError, SocketError) as e: print e print url log( "tpl_v3.2-output.txt", "error: " + str(e.message) + "\nURL: " + url + "\n") print "waiting 5 seconds..." time.sleep(5) database_lock = False except sqlite3.OperationalError: print("database locked, waiting 5 seconds...") log("tpl_v3.2-output.txt", "gglsbl3 database is locked") time.sleep(5) database_lock = False except KeyError: # 18 Jun 2018, disabling error: 'exceptions.KeyError'>, KeyError('matches',) # as appearing every minute or so print "Looks like a key error:", sys.exc_info()[1] log( "certstream-url-checker-v2-phishing_5.txt", "Looks like a key error: " + str(sys.exc_info()[1])) print "URL:", url time.sleep(5) database_lock = False except: print "We have an error:", sys.exc_info()[1] print sys.exc_info() import os import traceback from send_email import sendAdminAlert script_file_name = os.path.basename(__file__) error_string = "" for frame in traceback.extract_tb( sys.exc_info()[2]): fname, lineno, fn, text = frame error_string += "\nError in %s on line %d" % ( fname, lineno) print error_string sendAdminAlert( "Error (within URLLookup_v2 loop) in " + script_file_name, "Python script: " + script_file_name + "\nError reprted: " + str(sys.exc_info()[1]) + "\nLine:" + str(error_string)) print "waiting 90 seconds" time.sleep(90) database_lock = False else: print "URL already in gsb_full_hash_lookup" #j+=1 counter += 1 printCounter += 1 if (printCounter == 10): #progress_bar(counter, num_urls, '%s of %s (hash_prefix_counter: %s)' % (counter, num_urls, hash_prefix_counter)) progress_bar(counter, num_urls, '%s of %s' % (counter, num_urls)) printCounter = 0 return (phish_count, malware_count, phishing_matches)
time.sleep(30) except mdb.Error, e: print "We have an error:", sys.exc_info()[1] import os import sys import traceback from send_email import sendAdminAlert script_file_name = os.path.basename(__file__) error_string = "" for frame in traceback.extract_tb(sys.exc_info()[2]): fname, lineno, fn, text = frame error_string += "\nError in %s on line %d" % (fname, lineno) print error_string sendAdminAlert( "MySQL error in twitter stream sample", "Python script: " + script_file_name + "\nError reported: " + str(sys.exc_info()[1]) + "\nLine:" + str(error_string) + "\n\n" + (str(getTS()))) print "Will try to reconnect to MySQL server in 90 seconds..." print getTS() time.sleep(90) #con = mdb.connect(config['mysql']['host'], config['mysql']['username'], config['mysql']['password'], config['mysql']['database_5'],charset='utf8',cursorclass = mdb.cursors.SSCursor) #cursor = con.cursor() #con.autocommit(True) except: print "Unknown error...", sys.exc_info()[0] print getTS() cur = con.cursor() sql = "INSERT INTO twitter_stream_error_log(twitter_sample_stream, error_reported, notes) VALUES(%s, %s, %s)"
print getTS() time.sleep(90) except mdb.Error, e: print "We have an error:", sys.exc_info()[1] import os import sys import traceback from send_email import sendAdminAlert script_file_name = os.path.basename(__file__) error_string = "" for frame in traceback.extract_tb(sys.exc_info()[2]): fname, lineno, fn, text = frame error_string += "\nError in %s on line %d" % (fname, lineno) print error_string sendAdminAlert( "MySQL error in main twitter stream (i.e URL filter)", "Python script: " + script_file_name + "\nError reported: " + str(sys.exc_info()[1]) + "\nLine:" + str(error_string)) print "Will try to reconnect to MySQL server in 90 seconds..." print getTS() time.sleep(90) #con = mdb.connect(config['mysql']['host'], config['mysql']['username'], config['mysql']['password'], config['mysql']['database_5'],charset='utf8',cursorclass = mdb.cursors.SSCursor) #cursor = con.cursor() #con.autocommit(True) except: print "We have an error:", sys.exc_info()[1] import os import sys import traceback from send_email import sendAdminAlert