def dbUpdate_plainList(): plainlist = plainl.plainMenu() #print(plainlist) stats = 0 coll = dbconnect.opensourcelistsColl() print( "Inserting into the database information from the Plain file list ...\n" ) try: for key, value in plainlist.items(): print('indicator = ', key) print('Intel = ', value) if coll.find({'indicator': key}).count() > 0: print("Indicator already in database\n") #collid = coll.find({'indicator':key})[0]['_id'] intelsource = coll.find({'indicatior': key})[0]['intelsource'] newintelsource = [value['intelsource'], intelsource] print(newintelsource[0]) print(newintelsource[1]) #print (collid) ## ID OF COLLECTION RECORD #print(value['intelsource']) ## NEW INTEL SOURCE else: try: #data = {'indicator': key, 'type': value['Type'], 'intelsource': value['IntelSource'], 'date': value['Date'], 'notes':['']} #coll.insert(data) stats += 1 except Exception as e: print(key, " could not be inserted into the database!!", e) except Exception as e: print( "Could not update database with the information for the Plain file list", e) if stats == 0: print("\n") print( "No information was inserted into the Intel Feeds database. ¯\_(ツ)_/¯ \n" ) else: print("\n") print( stats, "new records were inserted into the database from your Plain file list.\n" )
def dbUpdate_opensourcelists(): stats = 0 coll = dbconnect.opensourcelistsColl() print( "Downloading and inserting into the database information from open source lists (Malc0de, Zeus Tracker, Locky, Bambenek and Emerging Threats) ...\n" ) try: for key, value in feeds.fetch_feeds().items(): #print ('IP = ', key) #print ('Intel = ', value) if coll.find({'indicator': key}).count() > 0: #print ("Indicator already in database\n") pass else: try: data = { 'indicator': key, 'type': value['Type'], 'intelsource': value['IntelSource'], 'date': value['Date'], 'notes': [''] } coll.insert(data) stats += 1 except Exception as e: print(key, " could not be inserted into the database!!", e) except Exception as e: print("Could not update database with Open Source Lists information", e) if stats == 0: print("\n") print( "No information was inserted into the Intel Feeds database. ¯\_(ツ)_/¯ \n" ) else: print("\n") print( stats, "new records were inserted into the database from open source lists (Malc0de, Zeus Tracker, Locky, Bambenek and Emerging Threats).\n" )
def stalkPrey(): opensourcedb = dbconnect.opensourcelistsColl() etpdb = dbconnect.feEtpColl() emailPattern = re.compile(r'[^@]+@[^@]+\.[^@]+') try: print ("\n") prey = input ("Input Prey: ") # Check if the input is an email address and focus the search on email addresses on FireEyeETP collection if re.match(emailPattern, prey): try: if etpdb.find({'from':prey}).count() == 1: # Print details if only 1 result results = etpdb.find({'from':prey}) print ("IntelSource : FireEye ETP") print ("FireETP Alert Number: ", results[0]['alert']) print ("Alert date: ", results[0]['time']) print ("Alert type: ", results[0]['type']) print ("File name or URL: ", results[0]['name']) print ("File Hash:", results[0]['md5']) print ("Sent from: ", results[0]['from']) print ("Sent to: ", results[0]['recipients']) print ("Email Subject: ", results[0]['subject']) print ("Notes or related IoCs: ", str(results[0]['evilips'])) # If more than one result (usually the case) gives the option to print all on raw format. elif etpdb.find({'from':prey}).count() > 1: print (etpdb.find({'from':prey}).count(), " preys were found!!\n") choise = input ("Do you want to stalk them all? [Y/N]: ") if (choise == 'Y') or (choise == 'y'): print (choise) results = etpdb.find({'from':prey}) for hit in results: print(hit) print("\n") else: pass except Exception as e: print (e) # IF the input is not an email, search threat intel collection for indicator elif opensourcedb.find({'indicator':prey}).count() > 0: results = opensourcedb.find({'indicator':prey}) print ("Prey found, stalking...\n") #print (results[0]['intelsource']) if results[0]['intelsource'] == 'FireEye_ETP': etpalertnumber = results[0]['notes'][0]['alert'] results = etpdb.find({'alert':etpalertnumber}) print ("IntelSource : FireEye ETP") print ("FireETP Alert Number: ", results[0]['alert']) print ("Alert date: ", results[0]['time']) print ("Alert type: ", results[0]['type']) print ("File name or URL: ", results[0]['name']) print ("File Hash:", results[0]['md5']) print ("Sent from: ", results[0]['from']) print ("Sent to: ", results[0]['recipients']) print ("Email Subject: ", results[0]['subject']) print ("Notes or related IoCs: ", str(results[0]['evilips'])) else: #for hit in results: # print(hit) print ("IntelSource: ", results[0]['intelsource']) print ("Type: ", results[0]['type']) print ("Date added to StalkerDB: ", results[0]['date']) else: print ("Stalking FAILED!! nothing found!!") print ("\n") input("Press Enter to Continue...") cls() except Exception as e: print ("Something is not right with that prey!", e)
def dbStatistics(): opensourcecoll = dbconnect.opensourcelistsColl() feetpcoll = dbconnect.feEtpColl() cls() print(" ######## FireEye ETP Database Collection Statistics #########\n") feetptotal = feetpcoll.count() print("Total number of record on FireEye ETP Collection: ", feetptotal) print("\n") typeurl = feetpcoll.find({'type': 'url'}).count() print("Total number of URL's on FireEye ETP Collection: ", typeurl) typedoc = feetpcoll.find({'type': 'doc'}).count() print("Total number of \"DOC\" files on FireEye ETP Collection: ", typedoc) typeexe = feetpcoll.find({'type': 'exe'}).count() print("Total number of \"EXE\" files on FireEye ETP Collection: ", typeexe) typezip = feetpcoll.find({'type': 'zip'}).count() print("Total number of \"ZIP\" files on FireEye ETP Collection: ", typezip) typejar = feetpcoll.find({'type': 'jar'}).count() print("Total number of \"JAR\" files on FireEye ETP Collection: ", typejar) typehtm = feetpcoll.find({'type': 'htm'}).count() print("Total number of \"HTM\" files on FireEye ETP Collection: ", typehtm) type7zip = feetpcoll.find({'type': '7zip'}).count() print("Total number of \"7ZIP\" files on FireEye ETP Collection: ", type7zip) typecom = feetpcoll.find({'type': 'com'}).count() print("Total number of \"COM\" files on FireEye ETP Collection: ", typecom) typepdf = feetpcoll.find({'type': 'pdf'}).count() print("Total number of \"PDF\" files on FireEye ETP Collection: ", typepdf) typedocx = feetpcoll.find({'type': 'docx'}).count() print("Total number of \"DOCX\" files on FireEye ETP Collection: ", typedocx) typexls = feetpcoll.find({'type': 'xls'}).count() print("Total number of \"XLS\" files on FireEye ETP Collection: ", typexls) typexlsx = feetpcoll.find({'type': 'xlsx'}).count() print("Total number of \"XLSX\" files on FireEye ETP Collection: ", typexlsx) typejs = feetpcoll.find({'type': 'js'}).count() print("Total number of \"JS\" files on FireEye ETP Collection: ", typejs) typevbs = feetpcoll.find({'type': 'vbs'}).count() print("Total number of \"VBS\" files on FireEye ETP Collection: ", typevbs) typeace = feetpcoll.find({'type': 'ace'}).count() print("Total number of \"ACE\" files on FireEye ETP Collection: ", typeace) typerar = feetpcoll.find({'type': 'rar'}).count() print("Total number of \"RAR\" files on FireEye ETP Collection: ", typerar) typebz2 = feetpcoll.find({'type': 'bz2'}).count() print("Total number of \"BZ2\" files on FireEye ETP Collection: ", typebz2) typebz = feetpcoll.find({'type': 'bz'}).count() print("Total number of \"BZ\" files on FireEye ETP Collection: ", typebz) typedocm = feetpcoll.find({'type': 'docm'}).count() print("Total number of \"DOCM\" files on FireEye ETP Collection: ", typedocm) typescr = feetpcoll.find({'type': 'scr'}).count() print("Total number of \"SCR\" files on FireEye ETP Collection: ", typescr) print("\n") print("####### OpenSource Threat Intel Collection Statistics #######\n") opensourcetotal = opensourcecoll.count() print("Total number of records on Open Source Collection: ", opensourcetotal) print("\n") typeip = opensourcecoll.find({'type': 'Intel::ADDR'}).count() print("Total number of IP addresses on Open Source Collection: ", typeip) typeurl = opensourcecoll.find({'type': 'Intel::DOMAIN'}).count() print("Total number of Domain Names on Open Source Collection: ", typeurl) typehash = opensourcecoll.find({'type': 'Intel::FILE_HASH'}).count() print("Total number of HASHES on Open Source Collection: ", typehash) print("\n") input("Press Enter to continue...") cls()
def dbUpdate_FireeyeETP(): coll = dbconnect.feEtpColl() # Connects to StalkerDB.etpalerts coll2 = dbconnect.opensourcelistsColl( ) # Connect to StalkerDB.opensourcelists today = datetime.datetime.now().strftime("%m-%d-%Y") stats = 0 etp_file = input( "Enter the name of the file containing ETP alerts in CSV form. (Include absolute path if file is not in Stalker folder): " ) if os.path.exists(etp_file): etpalerts = fireeye.readETP(etp_file) try: print("\n") print("Updating StalkerDB.etpalerts Collection....") for key, value in etpalerts.items(): if coll.find({'alert': key}).count() > 0: # Value already in database pass else: data = { 'alert': key, 'time': value['Time'], 'dbtime': today, 'from': value['From'], 'recipients': value['Recipients'], 'subject': value['Subject'], 'type': value['Type'], 'name': value['Name'], 'md5': value['MD5'], 'evilips': value['evilips'] } coll.insert(data) stats += 1 except Exception as e: print( "Something went wrong while updating \"etpalerts\" collection!", e) if stats == 0: print("\n") print( "Nothing new found. No information was inserted into the database. ¯\_(ツ)_/¯ \n" ) else: print("\n") print( stats, "new records were inserted into the database from the FireEye ETP alerts file.\n" ) ## End of importing data from alerts file into FireETP database. ## Next we will use the same information to update Intel Feeds database with ETP information. ## Key = ETP Alert number ## Value = { Time, From, Recipients, Subject, Type, Name "name of the binary file, or full URL", MD5, evilips[] } ## Now using coll2 to connect to opensourcelists collection on StalkerDB try: print("\n") print( "Updating StalkerDB.opensourcelists with ETP information.....") statshash = 0 statsurls = 0 statsunknown = 0 for key, value in etpalerts.items(): if value['Type'] == 'url': if coll2.find({'indicator': value['Name']}).count() > 0: pass # Value already in database else: # URL path comes as evilips for URLS from fireeye_export_list.py, key is teh ETP alert number data = { 'indicator': value['Name'], 'type': 'Intel::DOMAIN', 'intelsource': 'FireEye_ETP', 'date': today, 'notes': [{ 'alert': key }, { 'path': value['evilips'] }] } coll2.insert(data) #### Insert into the database statsurls += 1 elif value['MD5'] != 'N/A': if coll2.find({'indicator': value['MD5']}).count() > 0: pass # Value already in database else: # Key is the ETP alert number, Name is the file name, evilips is the IP addresses associated with the binary data = { 'indicator': value['MD5'], 'type': 'Intel::FILE_HASH', 'intelsource': 'FireEye_ETP', 'date': today, 'notes': [{ 'alert': key }, { 'filename': value['Name'] }, { 'evilips': value['evilips'] }] } coll2.insert(data) ### Insert into the database statshash += 1 else: statsunknown += 1 except Exception as e: print( "Something went wrong while updating StalkerDb.opensourcelists collection with ETP information.", e) if stats == 0: print("\n") print( "Nothing new found. No information was inserted into the database. ¯\_(ツ)_/¯ \n" ) elif statsunknown > 0: print("\n") print( "%d New URLs, and %d new Hashes were inserted into the Intel Feeds database from the FireEye ETP alerts file.\n" % (statsurls, statshash)) print(statsunknown, " unknown records were ignored! ¬_¬ ") else: print("\n") print( "%d New URLs, and %d new Hashes were inserted into the Intel Feeds database from the FireEye ETP alerts file.\n" % (statsurls, statshash)) else: # End of "if os.path.exists" print( "File not found. Make sure the file is on the Stalker folder, or use absolute path.\n" )