Exemplo n.º 1
0
	def getPOP(self, user, pwd, server):
		m = poplib.POP3_SSL(server)
		m.user(user)
		m.pass_(pwd)
		emailCount, total_bytes = m.stat()
		comment = "Tasking-"+server
		counter = 0
		log = "##INFO##, POP Connection to: %s Retrieving %s Emails in %s bytes" % (server, emailCount, total_bytes)
		MaildbLog.logEntry(log)
		for email in range(emailCount):
			counter +=1
			emailFile = os.path.join(transferDir, server+str(counter) + ".txt")
			msgFile = open(emailFile, "w")
			for msg in m.retr(email+1)[1]:				
				msgFile.write(msg)
				msgFile.write("\n")
			msgFile.close()
			
			try: # this try means an empty db file wont break it
				lastPath = db.lastLine()
				newPath = str(lastPath + 1) # will be used to set the database and match it to a physical location
			except:
				newPath = '1'

			reportDir = os.path.join(reportRoot, newPath)
			if not os.path.exists(reportDir):
				os.makedirs(reportDir) #Create the Dir Structure
				os.makedirs(os.path.join(reportDir, "attatchments"))
				shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml"))
			from core.parse import emlParse
			emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script
			log = "##INFO## Email Submitted With ID " + newPath
			MaildbLog.logEntry(log)
			parseRun = emlParse(emlName, reportDir, comment) # Call the parse script
			parseRun.run()
Exemplo n.º 2
0
	def getIMAP(self, user, pwd, server, inbox):

		m = imaplib.IMAP4_SSL(server)
		m.login(user,pwd)
		m.select(inbox)

		resp, items = m.search(None, "ALL") # IMAP Filter Rules here
		items = items[0].split()
		comment = "Tasking-"+inbox
		count = len(items)
		counter = 0
		log = "##INFO##, IMAP Connection to: %s Retrieving %s Emails " % (server, count)
		MaildbLog.logEntry(log)
		for emailid in items:
			emailFile = os.path.join(transferDir, inbox+emailid + ".txt")
			counter +=1
			resp, data = m.fetch(emailid, "(RFC822)")
			email_body = data[0][1]
			msgFile = open(emailFile, "w")
			msgFile.write(email_body)
			msgFile.close()
			lastPath = db.lastLine()
			try: # this try means an empty db file wont break it
				newPath = str(lastPath + 1) # will be used to set the database and match it to a physical location
			except:
				newPath = '1'

			reportDir = os.path.join(reportRoot, newPath)
			if not os.path.exists(reportDir):
				os.makedirs(reportDir) #Create the Dir Structure
				os.makedirs(os.path.join(reportDir, "attatchments"))
				shutil.copyfile(emailFile, os.path.join(reportDir, "message.eml"))
			from core.parse import emlParse
			emlName = os.path.join(reportDir, "message.eml") # Name of the eml to pass over to the parse script
			log = "##INFO## Email Submitted With ID " + newPath
			MaildbLog.logEntry(log)
			parseRun = emlParse(emlName, reportDir, comment) # Call the parse script
			parseRun.run()
Exemplo n.º 3
0
Arquivo: parse.py Projeto: hcit/Maildb
	def run(self):
		x = open(os.path.join(self.reportDir, self.msgFile))
		msg = email.message_from_file(x) # open the eml file so we can parse ie
		x.close()

		
		################# Header Information ###################
		# Get me all the sections then write them as one big sql line
		
		dateLine = msg.get('Date')
		
		msg_id = int(os.path.basename(self.reportDir)) # Unique id for this email used to cross ref other tables
		fromAdd = msg['from'] # might need to tidy this up a little bit using the address parse option
		stringIt = str(fromAdd)
		dbFrom = stringIt[stringIt.find("<")+1:stringIt.find(">")]
		# very messy need to fix this.
		addDomain = dbFrom[dbFrom.find("@")+1:] 
		subjectLine = msg['subject']
		x_mailer = msg['X-Mailer']
		x_priority = msg['X-Priority']
		try:
			message_id = re.sub('[<>]', '', msg['Message-ID'])
		except:
			message_id = msg['Message-ID']
		hops = msg.get_all('Received')
		if hops:
			for hop in hops:
				db.cursor.execute('INSERT INTO hops VALUES(?,?)', (msg_id, hop))
		try:
			sender = re.sub('[<>]', '', msg.get('From')) # remove <> so it renders correctly in the HTML
		except:
			sender = dbFrom
		try:
			to_add = re.sub('[<>]', '', msg.get('To')) #
		except:
			to_add = msg.get('To')
		try:
			cc_add = re.sub('[<>]', '', msg.get('cc'))
		except:
			cc_add = msg.get('cc')
		try:
			bcc_add = re.sub('[<>]', '', msg.get('Bcc'))
		except:
			bcc_add = msg.get('bcc')
		sqlHeader = ( msg_id, dateLine, sender, addDomain, subjectLine, x_mailer, x_priority, message_id, cc_add, bcc_add, to_add)
		db.cursor.execute('INSERT INTO header VALUES (?,?,?,?,?,?,?,?,?,?,?)', sqlHeader)
		db.conn.commit()
		
		
		################# ATTATCHMENTS ##########################
		## This section with thanks to the Python Docs
		
		counter = 0
		for part in msg.walk():
			if part.get_content_maintype() == 'multipart':				
				continue
			
			if part.get_content_type() == 'text/plain': # Plain Text Body				
				contents = part.get_payload(decode=True)
				links = re.findall(r'(https?://\S+)', contents)
				link_type = "url"
				for urls in links:
					db.cursor.execute('INSERT INTO links VALUES(?,?,?)', (msg_id, link_type, urls))
				db.conn.commit()				
				from core.cleanHtml import cleanHTML
				htmlStrip = cleanHTML().safe_html(contents)
				if htmlStrip is not None:
					fp = open(os.path.join(self.reportDir, "attatchments", "body.txt"), 'wb')
					fp.write(htmlStrip.encode('ascii', 'ignore'))
					fp.close()

				
			if part.get_content_type() == 'text/html': # HTML Body
				contents = part.get_payload(decode=True)
				soup = BeautifulSoup(contents)
				for link in soup.find_all('a'):
					link_type = "url"
					urls = link.get('href')
					db.cursor.execute('INSERT INTO links VALUES(?,?,?)', (msg_id, link_type, urls))
				for images in soup.find_all('img'):
					link_type = "img"
					image = images.get('src')
					db.cursor.execute('INSERT INTO links VALUES(?,?,?)', (msg_id, link_type, urls))
				for iframes in soup.find_all('iframe'):
					link_type = "iframe"
					db.cursor.execute('INSERT INTO links VALUES(?,?,?)', (msg_id, link_type, urls))
				db.conn.commit()				
				from core.cleanHtml import cleanHTML
				htmlStrip = cleanHTML().safe_html(contents)
				if htmlStrip is not None:
					fp = open(os.path.join(self.reportDir, "attatchments", "htmlbody.txt"), 'wb')
					fp.write(htmlStrip.encode('ascii', 'ignore'))
					fp.close()
 			
			if part.get('Content-Disposition') is None: # Actual File attatchments here
				continue
				
				
			from bs4 import UnicodeDammit
			filenameraw = str(part.get_filename())			
			dammit = UnicodeDammit(filenameraw)
			enctype = dammit.original_encoding
			if enctype == "ascii":
				filename = dammit.unicode_markup
			else:
				ext = mimetypes.guess_extension(part.get_content_type())
				filename = '%s-encoded-File-%s.%s' % (enctype, counter, ext)
				log = "##INFO##, Encoded File Created, " + os.path.join(self.reportDir, "attatchments", filenameraw)
				MaildbLog().logEntry(log)								
			if filename == 'None': # if theres no name then guess the extension and make something up
				ext = mimetypes.guess_extension(part.get_content_type())
				if not ext:
					ext = ".bin"
				filename = 'part-%03d%s' % (counter, ext)
			counter +=1
			fp = open(os.path.join(self.reportDir, "attatchments", filename), 'wb') # write the attatchment out to a folder
			# Deal With Zero Size Files
			if part.get_payload(decode=True) is None:
				part_data = "This is a Zero Byte File"
				fp.write(part_data)
				fp.close()
				log = "##INFO##, Zero Byte File Created, " + os.path.join(self.reportDir, "attatchments", filename)
				MaildbLog().logEntry(log)
			
			else:
				fp.write(part.get_payload(decode=True))
				fp.close()
				part_data = part.get_payload(decode=True)
			fileSize = os.path.getsize(os.path.join(self.reportDir, "attatchments", filename))
			fileExt = os.path.splitext(os.path.join(self.reportDir, "attatchments", filename))
		
			md5Hash = MailHash().HashMD5(part_data)
			sha256Hash = MailHash().HashSha256(part_data)
		
		
			if ssdeepcheck == '1': # check to see if users has enabled ssdeep
				try: 	#gracefull fail if the python wrapper is not installed.
				
					ssdHash = MailHash().Hashssdeep(part_data)
				except:
					log = "##INFO##, ssdeep not installed or config file not set"
					MaildbLog.logEntry(log)
					ssdHash = "0"
			else:
				ssdHash = "0"
			
			if enableYara =='1':
				from core.yarascan import Scan
				filetoScan = os.path.join(self.reportDir, "attatchments", filename)
				result = Scan().fileScan(filetoScan, md5Hash)
				match = '0'
				if result:
					yaraMatch = '3'
					match = '3'
				else:
					yaraMatch = '0'
					
			# database stuff here
		
			sqlAttatchments = (msg_id, str(filename), fileExt[1][1:], fileSize, md5Hash, sha256Hash, ssdHash, yaraMatch)
			try:
				msg_id = int(db.lastLine())
			except:
				msg_id = '1'
			db.cursor.execute('INSERT INTO attatch VALUES (?,?,?,?,?,?,?,?)', sqlAttatchments)
			db.cursor.execute("UPDATE main SET 'attCount'=?, 'Revmatch'=? WHERE msg_id=?", (counter, match, msg_id,))
			db.conn.commit()