def getPOP(self, user, pwd, server):
m = poplib.POP3_SSL(server)
m.user(user)
m.pass_(pwd)
emailCount, total_bytes = m.stat()
comment = "Tasking-"+server
counter = 0
log = "##INFO##, POP Connection to: %s Retrieving %s Emails in %s bytes" % (server, emailCount, total_bytes)
MaildbLog.logEntry(log)
for email in range(emailCount):
counter +=1
emailFile = os.path.join(transferDir, server+str(counter) + ".txt")
msgFile = open(emailFile, "w")
for msg in m.retr(email+1)[1]:
msgFile.write(msg)
msgFile.write("\n")
msgFile.close()
try: # this try means an empty db file wont break it
lastPath = db.lastLine()
newPath = str(lastPath + 1) # will be used to set the database and match it to a physical location
except:
newPath = '1'
reportDir = os.path.join(reportRoot, newPath)
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml"))
from core.parse import emlParse
emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script
log = "##INFO## Email Submitted With ID " + newPath
MaildbLog.logEntry(log)
parseRun = emlParse(emlName, reportDir, comment) # Call the parse script
parseRun.run()
def getIMAP(self, user, pwd, server, inbox): m = imaplib.IMAP4_SSL(server) m.login(user,pwd) m.select(inbox) resp, items = m.search(None, "ALL") # IMAP Filter Rules here items = items[0].split() comment = "Tasking-"+inbox count = len(items) counter = 0 log = "##INFO##, IMAP Connection to: %s Retrieving %s Emails " % (server, count) MaildbLog.logEntry(log) for emailid in items: emailFile = os.path.join(transferDir, inbox+emailid + ".txt") counter +=1 resp, data = m.fetch(emailid, "(RFC822)") email_body = data[0][1] msgFile = open(emailFile, "w") msgFile.write(email_body) msgFile.close() lastPath = db.lastLine() try: # this try means an empty db file wont break it newPath = str(lastPath + 1) # will be used to set the database and match it to a physical location except: newPath = '1' reportDir = os.path.join(reportRoot, newPath) if not os.path.exists(reportDir): os.makedirs(reportDir) #Create the Dir Structure os.makedirs(os.path.join(reportDir, "attatchments")) shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml")) from core.parse import emlParse emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script log = "##INFO## Email Submitted With ID " + newPath MaildbLog.logEntry(log) parseRun = emlParse(emlName, reportDir, comment) # Call the parse script parseRun.run()
def run(self):
x = open(os.path.join(self.reportDir, self.msgFile))
msg = email.message_from_file(x) # open the eml file so we can parse ie
x.close()
################# Header Information ###################
# Get me all the sections then write them as one big sql line
dateLine = msg.get('Date')
msg_id = int(os.path.basename(self.reportDir)) # Unique id for this email used to cross ref other tables
fromAdd = msg['from'] # might need to tidy this up a little bit using the address parse option
stringIt = str(fromAdd)
dbFrom = stringIt[stringIt.find("<")+1:stringIt.find(">")]
# very messy need to fix this.
addDomain = dbFrom[dbFrom.find("@")+1:]
subjectLine = msg['subject']
x_mailer = msg['X-Mailer']
x_priority = msg['X-Priority']
try:
message_id = re.sub('[<>]', '', msg['Message-ID'])
except:
message_id = msg['Message-ID']
hops = msg.get_all('Received')
if hops:
for hop in hops:
db.cursor.execute('INSERT INTO hops VALUES(?,?)', (msg_id, hop))
try:
sender = re.sub('[<>]', '', msg.get('From')) # remove <> so it renders correctly in the HTML
except:
sender = dbFrom
try:
to_add = re.sub('[<>]', '', msg.get('To')) #
except:
to_add = msg.get('To')
try:
cc_add = re.sub('[<>]', '', msg.get('cc'))
except:
cc_add = msg.get('cc')
try:
bcc_add = re.sub('[<>]', '', msg.get('Bcc'))
except:
bcc_add = msg.get('bcc')
sqlHeader = ( msg_id, dateLine, sender, addDomain, subjectLine, x_mailer, x_priority, message_id, cc_add, bcc_add, to_add)
db.cursor.execute('INSERT INTO header VALUES (?,?,?,?,?,?,?,?,?,?,?)', sqlHeader)
db.conn.commit()
################# ATTATCHMENTS ##########################
## This section with thanks to the Python Docs
counter = 0
for part in msg.walk():
if part.get_content_maintype() == 'multipart':
continue
if part.get_content_type() == 'text/plain': # Plain Text Body
contents = part.get_payload(decode=True)
links = re.findall(r'(https?://\S+)', contents)
link_type = "url"
for urls in links:
db.cursor.execute('INSERT INTO links VALUES(?,?,?)', (msg_id, link_type, urls))
db.conn.commit()
from core.cleanHtml import cleanHTML
htmlStrip = cleanHTML().safe_html(contents)
if htmlStrip is not None:
fp = open(os.path.join(self.reportDir, "attatchments", "body.txt"), 'wb')
fp.write(htmlStrip.encode('ascii', 'ignore'))
fp.close()
if part.get_content_type() == 'text/html': # HTML Body
contents = part.get_payload(decode=True)
soup = BeautifulSoup(contents)
for link in soup.find_all('a'):
link_type = "url"
urls = link.get('href')
db.cursor.execute('INSERT INTO links VALUES(?,?,?)', (msg_id, link_type, urls))
for images in soup.find_all('img'):
link_type = "img"
image = images.get('src')
db.cursor.execute('INSERT INTO links VALUES(?,?,?)', (msg_id, link_type, urls))
for iframes in soup.find_all('iframe'):
link_type = "iframe"
db.cursor.execute('INSERT INTO links VALUES(?,?,?)', (msg_id, link_type, urls))
db.conn.commit()
from core.cleanHtml import cleanHTML
htmlStrip = cleanHTML().safe_html(contents)
if htmlStrip is not None:
fp = open(os.path.join(self.reportDir, "attatchments", "htmlbody.txt"), 'wb')
fp.write(htmlStrip.encode('ascii', 'ignore'))
fp.close()
if part.get('Content-Disposition') is None: # Actual File attatchments here
continue
from bs4 import UnicodeDammit
filenameraw = str(part.get_filename())
dammit = UnicodeDammit(filenameraw)
enctype = dammit.original_encoding
if enctype == "ascii":
filename = dammit.unicode_markup
else:
ext = mimetypes.guess_extension(part.get_content_type())
filename = '%s-encoded-File-%s.%s' % (enctype, counter, ext)
log = "##INFO##, Encoded File Created, " + os.path.join(self.reportDir, "attatchments", filenameraw)
MaildbLog().logEntry(log)
if filename == 'None': # if theres no name then guess the extension and make something up
ext = mimetypes.guess_extension(part.get_content_type())
if not ext:
ext = ".bin"
filename = 'part-%03d%s' % (counter, ext)
counter +=1
fp = open(os.path.join(self.reportDir, "attatchments", filename), 'wb') # write the attatchment out to a folder
# Deal With Zero Size Files
if part.get_payload(decode=True) is None:
part_data = "This is a Zero Byte File"
fp.write(part_data)
fp.close()
log = "##INFO##, Zero Byte File Created, " + os.path.join(self.reportDir, "attatchments", filename)
MaildbLog().logEntry(log)
else:
fp.write(part.get_payload(decode=True))
fp.close()
part_data = part.get_payload(decode=True)
fileSize = os.path.getsize(os.path.join(self.reportDir, "attatchments", filename))
fileExt = os.path.splitext(os.path.join(self.reportDir, "attatchments", filename))
md5Hash = MailHash().HashMD5(part_data)
sha256Hash = MailHash().HashSha256(part_data)
if ssdeepcheck == '1': # check to see if users has enabled ssdeep
try: #gracefull fail if the python wrapper is not installed.
ssdHash = MailHash().Hashssdeep(part_data)
except:
log = "##INFO##, ssdeep not installed or config file not set"
MaildbLog.logEntry(log)
ssdHash = "0"
else:
ssdHash = "0"
if enableYara =='1':
from core.yarascan import Scan
filetoScan = os.path.join(self.reportDir, "attatchments", filename)
result = Scan().fileScan(filetoScan, md5Hash)
match = '0'
if result:
yaraMatch = '3'
match = '3'
else:
yaraMatch = '0'
# database stuff here
sqlAttatchments = (msg_id, str(filename), fileExt[1][1:], fileSize, md5Hash, sha256Hash, ssdHash, yaraMatch)
try:
msg_id = int(db.lastLine())
except:
msg_id = '1'
db.cursor.execute('INSERT INTO attatch VALUES (?,?,?,?,?,?,?,?)', sqlAttatchments)
db.cursor.execute("UPDATE main SET 'attCount'=?, 'Revmatch'=? WHERE msg_id=?", (counter, match, msg_id,))
db.conn.commit()
