def getPOP(self, user, pwd, server):
tmpDir = tempfile.mkdtemp()
m = poplib.POP3_SSL(server)
m.user(user)
m.pass_(pwd)
emailCount, total_bytes = m.stat()
date = datetime.now().strftime("%Y-%m-%d")
comment = "Tasking-"+date+"-"+server
counter = 0
for email in range(emailCount):
counter +=1
emailFile = os.path.join(tmpDir, server+str(counter) + ".txt")
msgFile = open(emailFile, "w")
for msg in m.retr(email+1)[1]:
msgFile.write(msg)
msgFile.write("\n")
msgFile.close()
newPath = db.lastLine()
reportDir = os.path.join(reportRoot, str(newPath))
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml"))
from core.parse import emlParse
emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script
parseRun = emlParse(emlName, reportDir, comment) # Call the parse script
parseRun.run()
shutil.rmtree(tmpDir)
def getInbox(self, user, pwd, server, inbox): m = imaplib.IMAP4_SSL(server) m.login(user,pwd) m.select(inbox) resp, items = m.search(None, "ALL") # IMAP Filter Rules here items = items[0].split() comment = "Tasking-IMAP" count = len(items) counter = 0 for emailid in items: emailFile = os.path.join(transferDir, server+emailid + ".txt") counter +=1 resp, data = m.fetch(emailid, "(RFC822)") email_body = data[0][1] msgFile = open(emailFile, "w") msgFile.write(email_body) msgFile.close() lastPath = db.lastLine() try: # this try means an empty db file wont break it newPath = str(lastPath + 1) # will be used to set the database and match it to a physical location except: newPath = '1' reportDir = os.path.join(reportRoot, newPath) if not os.path.exists(reportDir): os.makedirs(reportDir) #Create the Dir Structure os.makedirs(os.path.join(reportDir, "attatchments")) shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml")) from core.parse import emlParse emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script parseRun = emlParse(emlName, reportDir, comment) # Call the parse script parseRun.run()
def getPOP(self, user, pwd, server):
tmpDir = tempfile.mkdtemp()
m = poplib.POP3_SSL(server)
m.user(user)
m.pass_(pwd)
emailCount, total_bytes = m.stat()
date = datetime.now().strftime("%Y-%m-%d")
comment = "Tasking-" + date + "-" + server
counter = 0
for email in range(emailCount):
counter += 1
emailFile = os.path.join(tmpDir, server + str(counter) + ".txt")
msgFile = open(emailFile, "w")
for msg in m.retr(email + 1)[1]:
msgFile.write(msg)
msgFile.write("\n")
msgFile.close()
newPath = db.lastLine()
reportDir = os.path.join(reportRoot, str(newPath))
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
shutil.copyfile(emailFile,
os.path.join(reportDir, "message.eml"))
from core.parse import emlParse
emlName = os.path.join(
reportDir, "message.eml"
) # Name of the eml to pass over to the parse script
parseRun = emlParse(emlName, reportDir,
comment) # Call the parse script
parseRun.run()
shutil.rmtree(tmpDir)
def getIMAP(self, user, pwd, server, inbox):
tmpDir = tempfile.mkdtemp()
print tmpDir
m = imaplib.IMAP4_SSL(server)
m.login(user,pwd)
m.select(inbox)
resp, items = m.search(None, "ALL") # IMAP Filter Rules here
items = items[0].split()
date = datetime.now().strftime("%Y-%m-%d")
comment = "Tasking-"+date+"-"+server
count = len(items)
counter = 0
for emailid in items:
emailFile = os.path.join(tmpDir, inbox+emailid + ".txt")
print emailFile
counter +=1
resp, data = m.fetch(emailid, "(RFC822)")
email_body = data[0][1]
with open(emailFile, "w+") as msgFile:
msgFile.write(email_body)
newPath = db.lastLine()
reportDir = os.path.join(reportRoot, str(newPath))
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml"))
from core.parse import emlParse
emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script
parseRun = emlParse(emlName, reportDir, comment) # Call the parse script
parseRun.run()
shutil.rmtree(tmpDir)
def getPOP(self, user, pwd, server):
m = poplib.POP3_SSL(server)
m.user(user)
m.pass_(pwd)
emailCount, total_bytes = m.stat()
comment = "Tasking-"+server
counter = 0
log = "##INFO##, POP Connection to: %s Retrieving %s Emails in %s bytes" % (server, emailCount, total_bytes)
MaildbLog.logEntry(log)
for email in range(emailCount):
counter +=1
emailFile = os.path.join(transferDir, server+str(counter) + ".txt")
msgFile = open(emailFile, "w")
for msg in m.retr(email+1)[1]:
msgFile.write(msg)
msgFile.write("\n")
msgFile.close()
try: # this try means an empty db file wont break it
lastPath = db.lastLine()
newPath = str(lastPath + 1) # will be used to set the database and match it to a physical location
except:
newPath = '1'
reportDir = os.path.join(reportRoot, newPath)
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml"))
from core.parse import emlParse
emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script
log = "##INFO## Email Submitted With ID " + newPath
MaildbLog.logEntry(log)
parseRun = emlParse(emlName, reportDir, comment) # Call the parse script
parseRun.run()
def getIMAP(self, user, pwd, server, inbox):
tmpDir = tempfile.mkdtemp()
print tmpDir
m = imaplib.IMAP4_SSL(server)
m.login(user, pwd)
m.select(inbox)
resp, items = m.search(None, "ALL") # IMAP Filter Rules here
items = items[0].split()
date = datetime.now().strftime("%Y-%m-%d")
comment = "Tasking-" + date + "-" + server
count = len(items)
counter = 0
for emailid in items:
emailFile = os.path.join(tmpDir, inbox + emailid + ".txt")
print emailFile
counter += 1
resp, data = m.fetch(emailid, "(RFC822)")
email_body = data[0][1]
with open(emailFile, "w+") as msgFile:
msgFile.write(email_body)
newPath = db.lastLine()
reportDir = os.path.join(reportRoot, str(newPath))
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
shutil.copyfile(emailFile,
os.path.join(reportDir, "message.eml"))
from core.parse import emlParse
emlName = os.path.join(
reportDir, "message.eml"
) # Name of the eml to pass over to the parse script
parseRun = emlParse(emlName, reportDir,
comment) # Call the parse script
parseRun.run()
shutil.rmtree(tmpDir)
def submitpcap(tmpDir, pcapfile, comment): # for pcaps
shutil.copyfile(os.path.join(tmpDir, pcapfile),
os.path.join(tmpDir, "raw.pcap"))
retcode = subprocess.call("(cd %s && tcpflow -r %s)" %
(os.path.join(tmpDir), "raw.pcap"),
shell=True)
suffix = "00025"
filecount = 0
for i in os.listdir(tmpDir):
if i.endswith(suffix): # i only want emails here
###SMTP Headers break the parser so remove them
filecount += 1
edit = open(os.path.join(tmpDir, i))
lines = edit.readlines()
edit.close()
new = open(os.path.join(tmpDir, "newFile.eml"), "w")
edited = 0
flag = 1
for line in lines:
if line.startswith("EHLO") or line.startswith("220 "):
flag = 0
if line.startswith("DATA") or line.startswith(
"354 Start") or line.startswith("354 Enter mail"):
flag = 1
if flag and not (line.startswith("DATA")
or line.startswith("354 Start")
or line.startswith("354 Enter mail")
or line.startswith("250 ")):
new.write(line)
new.close()
newFile = "newFile.eml"
from core.parse import emlParse
lastPath = db.lastLine()
newPath = str(lastPath)
logging.info('Record %s Submitted', newPath)
emlName = os.path.basename(
i) # Name of the eml to pass oveflogr to the parse script
reportDir = os.path.join(reportRoot, newPath)
logging.info('Pcap Submitted')
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
if os.path.getsize(os.path.join(
tmpDir,
newFile)) > 200: # if file is this small it has no data
shutil.copyfile(
os.path.join(tmpDir, newFile),
os.path.join(reportDir,
"message.eml")) #copy the message in as is
parseRun = emlParse(i, reportDir,
comment) # Call the parse script
parseRun.run()
def submitpcap(self, pcapfile, comment):
if not os.path.exists(os.path.join(MaildbRoot, "tmp")):
os.mkdir(os.path.join(MaildbRoot, "tmp"))
shutil.copyfile(os.path.join(MaildbRoot, "tmp", pcapfile), os.path.join(transferDir, "raw.pcap"))
retcode = subprocess.call("(cd %s && tcpflow -r %s)"%(os.path.join(MaildbRoot, "tmp"), "raw.pcap"), shell=True)
suffix = "00025"
filecount = 0
for i in os.listdir(transferDir):
if i.endswith(suffix): # i only want emails here
###SMTP Headers break the parser so remove them
filecount +=1
edit = open(os.path.join(transferDir, i))
lines = edit.readlines()
edit.close()
new = open(os.path.join(transferDir, "newFile.eml"), "w")
edited = 0
flag = 1
for line in lines:
if line.startswith("EHLO") or line.startswith("220 "):
flag = 0
if line.startswith("DATA") or line.startswith("354 Start") or line.startswith("354 Enter mail"):
flag = 1
if flag and not (line.startswith("DATA") or line.startswith("354 Start") or line.startswith("354 Enter mail") or line.startswith("250 ")):
new.write(line)
new.close()
newFile = "newFile.eml"
from core.parse import emlParse
try: # this try means an empty db file wont break it
lastPath = db.lastLine()
newPath = str(lastPath + 1) # will be used to set the database and match it to a physical location
except:
newPath = '1'
emlName = os.path.basename(i) # Name of the eml to pass over to the parse script
reportDir = os.path.join(reportRoot, newPath)
log = "##INFO##, PCAP Submitted With ID " + newPath
writeLog.logEntry(log)
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
if os.path.getsize(os.path.join(transferDir, newFile)) > 200: # if file is this small it has no data
shutil.copyfile(os.path.join(transferDir, newFile), os.path.join(reportDir, "message.eml")) #copy the message in as is
parseRun = emlParse(i, reportDir, comment) # Call the parse script
parseRun.run()
def submit(self, comment):
for emlfile in os.listdir(transferDir): #Run for each email file
from core.parse import emlParse
print emlfile
try: # this try means an empty db file wont break it
lastPath = db.lastLine()
newPath = str(lastPath + 1) # will be used to set the database and match it to a physical location
except:
newPath = '1'
log = "##INFO## Email Submitted With ID " + newPath
writeLog.logEntry(log)
reportDir = os.path.join(reportRoot, newPath)
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
# SMTP Headers break the parser so remove them
edit = open(os.path.join(MaildbRoot, "tmp", emlfile))
lines = edit.readlines()
edit.close()
flag = 1
edited = 0
new = open(os.path.join(MaildbRoot, "tmp", "newFile.eml"), "w")
for line in lines:
if line.startswith("EHLO") or line.startswith("220 "):
flag = 0
edited = 1
if line.startswith("DATA") or line.startswith("354 Start") or line.startswith("354 Enter mail"):
flag = 1
if flag and not (line.startswith("DATA") or line.startswith("354 Start") or line.startswith("354 Enter mail") or line.startswith("250 ")):
new.write(line)
new.close()
if edited:
emailFile = os.path.join(transferDir, "newFile.eml")
print "Copy Edited"
else:
emailFile = os.path.join(transferDir, emlfile)
print "copy Non Edited"
shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml"))
emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script
parseRun = emlParse(emlName, reportDir, comment) # Call the parse script
parseRun.run()
return newPath
def submit(tmpDir, comment): # THis is for txt files
for emlfile in os.listdir(tmpDir): #Run for each email file
from core.parse import emlParse
lastPath = db.lastLine()
newPath = str(lastPath)
logging.info('Record %s Submitted', newPath)
reportDir = os.path.join(reportRoot, newPath)
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
# SMTP Headers break the parser so remove them
edit = open(os.path.join(tmpDir, emlfile))
lines = edit.readlines()
edit.close()
flag = 1
edited = 0
new = open(os.path.join(tmpDir, "newFile.eml"), "w")
for line in lines:
if line.startswith("EHLO") or line.startswith("220 "):
flag = 0
edited = 1
if line.startswith("DATA") or line.startswith(
"354 Start") or line.startswith("354 Enter mail"):
flag = 1
if flag and not (line.startswith("DATA")
or line.startswith("354 Start")
or line.startswith("354 Enter mail")
or line.startswith("250 ")):
new.write(line)
new.close()
if edited:
emailFile = os.path.join(tmpDir, "newFile.eml")
else:
emailFile = os.path.join(tmpDir, emlfile)
shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml"))
emlName = os.path.join(
reportDir,
"message.eml") # Name of the eml to pass over to the parse script
parseRun = emlParse(emlName, reportDir, comment) # Call the parse script
parseRun.run()
return newPath
def submitpcap(tmpDir, pcapfile, comment): # for pcaps
shutil.copyfile(os.path.join(tmpDir, pcapfile), os.path.join(tmpDir, "raw.pcap"))
retcode = subprocess.call("(cd %s && tcpflow -r %s)"%(os.path.join(tmpDir), "raw.pcap"), shell=True)
suffix = "00025"
filecount = 0
for i in os.listdir(tmpDir):
if i.endswith(suffix): # i only want emails here
###SMTP Headers break the parser so remove them
filecount +=1
edit = open(os.path.join(tmpDir, i))
lines = edit.readlines()
edit.close()
new = open(os.path.join(tmpDir, "newFile.eml"), "w")
edited = 0
flag = 1
for line in lines:
if line.startswith("EHLO") or line.startswith("220 "):
flag = 0
if line.startswith("DATA") or line.startswith("354 Start") or line.startswith("354 Enter mail"):
flag = 1
if flag and not (line.startswith("DATA") or line.startswith("354 Start") or line.startswith("354 Enter mail") or line.startswith("250 ")):
new.write(line)
new.close()
newFile = "newFile.eml"
from core.parse import emlParse
lastPath = db.lastLine()
newPath = str(lastPath)
logging.info('Record %s Submitted', newPath)
emlName = os.path.basename(i) # Name of the eml to pass oveflogr to the parse script
reportDir = os.path.join(reportRoot, newPath)
logging.info('Pcap Submitted')
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
if os.path.getsize(os.path.join(tmpDir, newFile)) > 200: # if file is this small it has no data
shutil.copyfile(os.path.join(tmpDir, newFile), os.path.join(reportDir, "message.eml")) #copy the message in as is
parseRun = emlParse(i, reportDir, comment) # Call the parse script
parseRun.run()
def getInbox(self, user, pwd, server, inbox):
m = imaplib.IMAP4_SSL(server)
m.login(user, pwd)
m.select(inbox)
resp, items = m.search(None, "ALL") # IMAP Filter Rules here
items = items[0].split()
comment = "Tasking-IMAP"
count = len(items)
counter = 0
for emailid in items:
emailFile = os.path.join(transferDir, server + emailid + ".txt")
counter += 1
resp, data = m.fetch(emailid, "(RFC822)")
email_body = data[0][1]
msgFile = open(emailFile, "w")
msgFile.write(email_body)
msgFile.close()
lastPath = db.lastLine()
try: # this try means an empty db file wont break it
newPath = str(
lastPath + 1
) # will be used to set the database and match it to a physical location
except:
newPath = '1'
reportDir = os.path.join(reportRoot, newPath)
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
shutil.copyfile(emailFile,
os.path.join(reportDir, "message.eml"))
from core.parse import emlParse
emlName = os.path.join(
reportDir, "message.eml"
) # Name of the eml to pass over to the parse script
parseRun = emlParse(emlName, reportDir,
comment) # Call the parse script
parseRun.run()
def submit(tmpDir, comment):# THis is for txt files
for emlfile in os.listdir(tmpDir): #Run for each email file
from core.parse import emlParse
lastPath = db.lastLine()
newPath = str(lastPath)
logging.info('Record %s Submitted', newPath)
reportDir = os.path.join(reportRoot, newPath)
if not os.path.exists(reportDir):
os.makedirs(reportDir) #Create the Dir Structure
os.makedirs(os.path.join(reportDir, "attatchments"))
# SMTP Headers break the parser so remove them
edit = open(os.path.join(tmpDir, emlfile))
lines = edit.readlines()
edit.close()
flag = 1
edited = 0
new = open(os.path.join(tmpDir, "newFile.eml"), "w")
for line in lines:
if line.startswith("EHLO") or line.startswith("220 "):
flag = 0
edited = 1
if line.startswith("DATA") or line.startswith("354 Start") or line.startswith("354 Enter mail"):
flag = 1
if flag and not (line.startswith("DATA") or line.startswith("354 Start") or line.startswith("354 Enter mail") or line.startswith("250 ")):
new.write(line)
new.close()
if edited:
emailFile = os.path.join(tmpDir, "newFile.eml")
else:
emailFile = os.path.join(tmpDir, emlfile)
shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml"))
emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script
parseRun = emlParse(emlName, reportDir, comment) # Call the parse script
parseRun.run()
return newPath
