def dbUpdate_plainList():
plainlist = plainl.plainMenu()
#print(plainlist)
stats = 0
coll = dbconnect.opensourcelistsColl()
print(
"Inserting into the database information from the Plain file list ...\n"
)
try:
for key, value in plainlist.items():
print('indicator = ', key)
print('Intel = ', value)
if coll.find({'indicator': key}).count() > 0:
print("Indicator already in database\n")
#collid = coll.find({'indicator':key})[0]['_id']
intelsource = coll.find({'indicatior': key})[0]['intelsource']
newintelsource = [value['intelsource'], intelsource]
print(newintelsource[0])
print(newintelsource[1])
#print (collid) ## ID OF COLLECTION RECORD
#print(value['intelsource']) ## NEW INTEL SOURCE
else:
try:
#data = {'indicator': key, 'type': value['Type'], 'intelsource': value['IntelSource'], 'date': value['Date'], 'notes':['']}
#coll.insert(data)
stats += 1
except Exception as e:
print(key, " could not be inserted into the database!!", e)
except Exception as e:
print(
"Could not update database with the information for the Plain file list",
e)
if stats == 0:
print("\n")
print(
"No information was inserted into the Intel Feeds database. ¯\_(ツ)_/¯ \n"
)
else:
print("\n")
print(
stats,
"new records were inserted into the database from your Plain file list.\n"
)
def dbUpdate_opensourcelists():
stats = 0
coll = dbconnect.opensourcelistsColl()
print(
"Downloading and inserting into the database information from open source lists (Malc0de, Zeus Tracker, Locky, Bambenek and Emerging Threats) ...\n"
)
try:
for key, value in feeds.fetch_feeds().items():
#print ('IP = ', key)
#print ('Intel = ', value)
if coll.find({'indicator': key}).count() > 0:
#print ("Indicator already in database\n")
pass
else:
try:
data = {
'indicator': key,
'type': value['Type'],
'intelsource': value['IntelSource'],
'date': value['Date'],
'notes': ['']
}
coll.insert(data)
stats += 1
except Exception as e:
print(key, " could not be inserted into the database!!", e)
except Exception as e:
print("Could not update database with Open Source Lists information",
e)
if stats == 0:
print("\n")
print(
"No information was inserted into the Intel Feeds database. ¯\_(ツ)_/¯ \n"
)
else:
print("\n")
print(
stats,
"new records were inserted into the database from open source lists (Malc0de, Zeus Tracker, Locky, Bambenek and Emerging Threats).\n"
)
def stalkPrey():
opensourcedb = dbconnect.opensourcelistsColl()
etpdb = dbconnect.feEtpColl()
emailPattern = re.compile(r'[^@]+@[^@]+\.[^@]+')
try:
print ("\n")
prey = input ("Input Prey: ")
# Check if the input is an email address and focus the search on email addresses on FireEyeETP collection
if re.match(emailPattern, prey):
try:
if etpdb.find({'from':prey}).count() == 1:
# Print details if only 1 result
results = etpdb.find({'from':prey})
print ("IntelSource : FireEye ETP")
print ("FireETP Alert Number: ", results[0]['alert'])
print ("Alert date: ", results[0]['time'])
print ("Alert type: ", results[0]['type'])
print ("File name or URL: ", results[0]['name'])
print ("File Hash:", results[0]['md5'])
print ("Sent from: ", results[0]['from'])
print ("Sent to: ", results[0]['recipients'])
print ("Email Subject: ", results[0]['subject'])
print ("Notes or related IoCs: ", str(results[0]['evilips']))
# If more than one result (usually the case) gives the option to print all on raw format.
elif etpdb.find({'from':prey}).count() > 1:
print (etpdb.find({'from':prey}).count(), " preys were found!!\n")
choise = input ("Do you want to stalk them all? [Y/N]: ")
if (choise == 'Y') or (choise == 'y'):
print (choise)
results = etpdb.find({'from':prey})
for hit in results:
print(hit)
print("\n")
else:
pass
except Exception as e: print (e)
# IF the input is not an email, search threat intel collection for indicator
elif opensourcedb.find({'indicator':prey}).count() > 0:
results = opensourcedb.find({'indicator':prey})
print ("Prey found, stalking...\n")
#print (results[0]['intelsource'])
if results[0]['intelsource'] == 'FireEye_ETP':
etpalertnumber = results[0]['notes'][0]['alert']
results = etpdb.find({'alert':etpalertnumber})
print ("IntelSource : FireEye ETP")
print ("FireETP Alert Number: ", results[0]['alert'])
print ("Alert date: ", results[0]['time'])
print ("Alert type: ", results[0]['type'])
print ("File name or URL: ", results[0]['name'])
print ("File Hash:", results[0]['md5'])
print ("Sent from: ", results[0]['from'])
print ("Sent to: ", results[0]['recipients'])
print ("Email Subject: ", results[0]['subject'])
print ("Notes or related IoCs: ", str(results[0]['evilips']))
else:
#for hit in results:
# print(hit)
print ("IntelSource: ", results[0]['intelsource'])
print ("Type: ", results[0]['type'])
print ("Date added to StalkerDB: ", results[0]['date'])
else:
print ("Stalking FAILED!! nothing found!!")
print ("\n")
input("Press Enter to Continue...")
cls()
except Exception as e: print ("Something is not right with that prey!", e)
def dbStatistics():
opensourcecoll = dbconnect.opensourcelistsColl()
feetpcoll = dbconnect.feEtpColl()
cls()
print(" ######## FireEye ETP Database Collection Statistics #########\n")
feetptotal = feetpcoll.count()
print("Total number of record on FireEye ETP Collection: ", feetptotal)
print("\n")
typeurl = feetpcoll.find({'type': 'url'}).count()
print("Total number of URL's on FireEye ETP Collection: ", typeurl)
typedoc = feetpcoll.find({'type': 'doc'}).count()
print("Total number of \"DOC\" files on FireEye ETP Collection: ", typedoc)
typeexe = feetpcoll.find({'type': 'exe'}).count()
print("Total number of \"EXE\" files on FireEye ETP Collection: ", typeexe)
typezip = feetpcoll.find({'type': 'zip'}).count()
print("Total number of \"ZIP\" files on FireEye ETP Collection: ", typezip)
typejar = feetpcoll.find({'type': 'jar'}).count()
print("Total number of \"JAR\" files on FireEye ETP Collection: ", typejar)
typehtm = feetpcoll.find({'type': 'htm'}).count()
print("Total number of \"HTM\" files on FireEye ETP Collection: ", typehtm)
type7zip = feetpcoll.find({'type': '7zip'}).count()
print("Total number of \"7ZIP\" files on FireEye ETP Collection: ",
type7zip)
typecom = feetpcoll.find({'type': 'com'}).count()
print("Total number of \"COM\" files on FireEye ETP Collection: ", typecom)
typepdf = feetpcoll.find({'type': 'pdf'}).count()
print("Total number of \"PDF\" files on FireEye ETP Collection: ", typepdf)
typedocx = feetpcoll.find({'type': 'docx'}).count()
print("Total number of \"DOCX\" files on FireEye ETP Collection: ",
typedocx)
typexls = feetpcoll.find({'type': 'xls'}).count()
print("Total number of \"XLS\" files on FireEye ETP Collection: ", typexls)
typexlsx = feetpcoll.find({'type': 'xlsx'}).count()
print("Total number of \"XLSX\" files on FireEye ETP Collection: ",
typexlsx)
typejs = feetpcoll.find({'type': 'js'}).count()
print("Total number of \"JS\" files on FireEye ETP Collection: ", typejs)
typevbs = feetpcoll.find({'type': 'vbs'}).count()
print("Total number of \"VBS\" files on FireEye ETP Collection: ", typevbs)
typeace = feetpcoll.find({'type': 'ace'}).count()
print("Total number of \"ACE\" files on FireEye ETP Collection: ", typeace)
typerar = feetpcoll.find({'type': 'rar'}).count()
print("Total number of \"RAR\" files on FireEye ETP Collection: ", typerar)
typebz2 = feetpcoll.find({'type': 'bz2'}).count()
print("Total number of \"BZ2\" files on FireEye ETP Collection: ", typebz2)
typebz = feetpcoll.find({'type': 'bz'}).count()
print("Total number of \"BZ\" files on FireEye ETP Collection: ", typebz)
typedocm = feetpcoll.find({'type': 'docm'}).count()
print("Total number of \"DOCM\" files on FireEye ETP Collection: ",
typedocm)
typescr = feetpcoll.find({'type': 'scr'}).count()
print("Total number of \"SCR\" files on FireEye ETP Collection: ", typescr)
print("\n")
print("####### OpenSource Threat Intel Collection Statistics #######\n")
opensourcetotal = opensourcecoll.count()
print("Total number of records on Open Source Collection: ",
opensourcetotal)
print("\n")
typeip = opensourcecoll.find({'type': 'Intel::ADDR'}).count()
print("Total number of IP addresses on Open Source Collection: ", typeip)
typeurl = opensourcecoll.find({'type': 'Intel::DOMAIN'}).count()
print("Total number of Domain Names on Open Source Collection: ", typeurl)
typehash = opensourcecoll.find({'type': 'Intel::FILE_HASH'}).count()
print("Total number of HASHES on Open Source Collection: ", typehash)
print("\n")
input("Press Enter to continue...")
cls()
def dbUpdate_FireeyeETP():
coll = dbconnect.feEtpColl() # Connects to StalkerDB.etpalerts
coll2 = dbconnect.opensourcelistsColl(
) # Connect to StalkerDB.opensourcelists
today = datetime.datetime.now().strftime("%m-%d-%Y")
stats = 0
etp_file = input(
"Enter the name of the file containing ETP alerts in CSV form. (Include absolute path if file is not in Stalker folder): "
)
if os.path.exists(etp_file):
etpalerts = fireeye.readETP(etp_file)
try:
print("\n")
print("Updating StalkerDB.etpalerts Collection....")
for key, value in etpalerts.items():
if coll.find({'alert': key}).count() > 0:
# Value already in database
pass
else:
data = {
'alert': key,
'time': value['Time'],
'dbtime': today,
'from': value['From'],
'recipients': value['Recipients'],
'subject': value['Subject'],
'type': value['Type'],
'name': value['Name'],
'md5': value['MD5'],
'evilips': value['evilips']
}
coll.insert(data)
stats += 1
except Exception as e:
print(
"Something went wrong while updating \"etpalerts\" collection!",
e)
if stats == 0:
print("\n")
print(
"Nothing new found. No information was inserted into the database. ¯\_(ツ)_/¯ \n"
)
else:
print("\n")
print(
stats,
"new records were inserted into the database from the FireEye ETP alerts file.\n"
)
## End of importing data from alerts file into FireETP database.
## Next we will use the same information to update Intel Feeds database with ETP information.
## Key = ETP Alert number
## Value = { Time, From, Recipients, Subject, Type, Name "name of the binary file, or full URL", MD5, evilips[] }
## Now using coll2 to connect to opensourcelists collection on StalkerDB
try:
print("\n")
print(
"Updating StalkerDB.opensourcelists with ETP information.....")
statshash = 0
statsurls = 0
statsunknown = 0
for key, value in etpalerts.items():
if value['Type'] == 'url':
if coll2.find({'indicator': value['Name']}).count() > 0:
pass
# Value already in database
else:
# URL path comes as evilips for URLS from fireeye_export_list.py, key is teh ETP alert number
data = {
'indicator': value['Name'],
'type': 'Intel::DOMAIN',
'intelsource': 'FireEye_ETP',
'date': today,
'notes': [{
'alert': key
}, {
'path': value['evilips']
}]
}
coll2.insert(data) #### Insert into the database
statsurls += 1
elif value['MD5'] != 'N/A':
if coll2.find({'indicator': value['MD5']}).count() > 0:
pass
# Value already in database
else:
# Key is the ETP alert number, Name is the file name, evilips is the IP addresses associated with the binary
data = {
'indicator':
value['MD5'],
'type':
'Intel::FILE_HASH',
'intelsource':
'FireEye_ETP',
'date':
today,
'notes': [{
'alert': key
}, {
'filename': value['Name']
}, {
'evilips': value['evilips']
}]
}
coll2.insert(data) ### Insert into the database
statshash += 1
else:
statsunknown += 1
except Exception as e:
print(
"Something went wrong while updating StalkerDb.opensourcelists collection with ETP information.",
e)
if stats == 0:
print("\n")
print(
"Nothing new found. No information was inserted into the database. ¯\_(ツ)_/¯ \n"
)
elif statsunknown > 0:
print("\n")
print(
"%d New URLs, and %d new Hashes were inserted into the Intel Feeds database from the FireEye ETP alerts file.\n"
% (statsurls, statshash))
print(statsunknown, " unknown records were ignored! ¬_¬ ")
else:
print("\n")
print(
"%d New URLs, and %d new Hashes were inserted into the Intel Feeds database from the FireEye ETP alerts file.\n"
% (statsurls, statshash))
else: # End of "if os.path.exists"
print(
"File not found. Make sure the file is on the Stalker folder, or use absolute path.\n"
)