def get_messages_in_folder(self, folder):
"""
Return a list of message dicts in *folder*.
Message dict format:
- uid
- flags
- email.email encoded RFC822 message
"""
if folder in self.list_folders():
self.server.select_folder(folder)
message_ids = self.server.search(['NOT DELETED'])
raw_messages = self.server.fetch(message_ids,['FLAGS','RFC822'])
messages = []
for uid in raw_messages.keys():
try:
messages.append({'uid': uid,
'folder': folder,
'flags': raw_messages[uid]['FLAGS'],
'email': email.message_from_string(raw_messages[uid]['RFC822'])
})
except:
pass
return messages
return []
def get_messages_in_folder(self, folder):
"""
Return a list of message dicts in *folder*.
Message dict format:
- uid
- flags
- email.email encoded RFC822 message
"""
if folder in self.list_folders():
self.server.select_folder(folder)
message_ids = self.server.search(['NOT DELETED'])
raw_messages = self.server.fetch(message_ids, ['FLAGS', 'RFC822'])
messages = []
for uid in raw_messages.keys():
try:
messages.append({
'uid':
uid,
'folder':
folder,
'flags':
raw_messages[uid]['FLAGS'],
'email':
email.message_from_string(raw_messages[uid]['RFC822'])
})
except:
pass
return messages
return []
def read_new_mail():
(retcode, messages) = conn.search(None, '(UNSEEN)')
temp = ''
if retcode == 'OK' and messages != ['']:
for num in messages[0].split(' '):
typ, data = conn.fetch(num, '(RFC822)')
msg = email.message_from_string(data[0][1])
conn.store(num, '+FLAGS', '\\Seen')
temp += msg.get_payload(decode=True)
print_order(temp)
def parse_email(config_path):
json_data = open(os.path.join('data/', 'imap_sample.txt'))
msg = json_data.read()
json_data.close()
e = email.message_from_string(msg)
for part in e.walk():
ct = part.get_params('Content-Type')
if ct and ('text/html', '') in ct:
html = part.get_payload(decode=True)
print(html)
soup = BeautifulSoup(html)
aa = soup.find_all('a', class_='proflink')
for a in aa:
print(a)
def get_message(self, folder, uid):
"""
Return a dict object for a given message in a given folder.
Message dict format:
- uid
- flags
- email.email encoded RFC822 message
"""
if folder in self.list_folders():
select_info = self.server.select_folder(folder)
raw_message = self.server.fetch(uid,['FLAGS','RFC822'])[uid]
return {'uid':uid,
'folder':folder,
'flags':raw_message['FLAGS'],
'email':email.message_from_string(raw_message['RFC822'])}
def process_msgs(self):
self.imap.select('Inbox')
status, data = self.imap.search(None, 'ALL')
for num in reversed(data[0].split()):
status, data = self.imap.fetch(num, '(RFC822)')
#print 'Message %s\n%s\n' % (num, data[0][1])
for response_part in data:
if isinstance(response_part, tuple):
msg = email.message_from_string(response_part[1])
varSubject = msg['subject']
varFrom = msg['from']
#remove the brackets around the sender email address
varFrom = varFrom.replace('<', '')
varFrom = varFrom.replace('>', '')
body = ""
for part in msg.walk():
### if part.get_content_type() == "text/plain": # ignore attachments / html
### body = body + part.get_payload(decode=True)
# is this part an attachment ?
if part.get('Content-Disposition') is None:
continue
filename = part.get_filename()
print("filename", filename)
counter = 1
# if there is no filename, we create one with a counter to avoid duplicates
if not filename:
filename = 'part-%03d%s' % (counter, 'bin')
counter += 1
file_list = filename.split('/')
file_list_len = len(file_list)
filename = file_list[file_list_len - 1]
att_path = "email_data/" + filename
#Check if its already there
if not os.path.isfile(att_path):
#print("writing file")
# finally write the stuff
fp = open(att_path, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
#process_input_message( varFrom,varSubject,body)
process_downloaded_messages()
def process_msgs(self):
self.imap.select('Inbox')
status, data = self.imap.search(None, 'ALL')
for num in reversed(data[0].split()):
status, data = self.imap.fetch(num, '(RFC822)')
#print 'Message %s\n%s\n' % (num, data[0][1])
for response_part in data:
if isinstance(response_part, tuple):
msg = email.message_from_string(response_part[1])
varSubject = msg['subject']
varFrom = msg['from']
#remove the brackets around the sender email address
varFrom = varFrom.replace('<', '')
varFrom = varFrom.replace('>', '')
body = ""
for part in msg.walk():
### if part.get_content_type() == "text/plain": # ignore attachments / html
### body = body + part.get_payload(decode=True)
# is this part an attachment ?
if part.get('Content-Disposition') is None:
continue
filename = part.get_filename()
print("filename",filename)
counter = 1
# if there is no filename, we create one with a counter to avoid duplicates
if not filename:
filename = 'part-%03d%s' % (counter, 'bin')
counter += 1
file_list = filename.split('/')
file_list_len = len(file_list)
filename = file_list[file_list_len-1]
att_path = "email_data/"+filename
#Check if its already there
if not os.path.isfile(att_path) :
#print("writing file")
# finally write the stuff
fp = open(att_path, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
#process_input_message( varFrom,varSubject,body)
process_downloaded_messages()
def get_message(self, folder, uid):
"""
Return a dict object for a given message in a given folder.
Message dict format:
- uid
- flags
- email.email encoded RFC822 message
"""
if folder in self.list_folders():
select_info = self.server.select_folder(folder)
raw_message = self.server.fetch(uid, ['FLAGS', 'RFC822'])[uid]
return {
'uid': uid,
'folder': folder,
'flags': raw_message['FLAGS'],
'email': email.message_from_string(raw_message['RFC822'])
}
def showNewMailMessages(self):
debugMsg('showNewMailMessages() entered')
#get IDs of all UNSEEN messages
typ, data = self.imap.SEARCH(None, 'UNSEEN')
debugMsg('data - new mail IDs:')
debugMsg(data, 0)
for id in data[0].split():
if not id in self.knownAboutMail:
#get the entire message
etyp, edata = self.imap.FETCH(id, '(RFC822)')
raw = edata[0][1] #puts message from list into string
msg = email.message_from_string(
raw
) # converts string to instance of message msg is an email message so multipart and walk work on it.
body = ''
#Finds the plain text version of the body of the message.
if msg.get_content_maintype(
) == 'multipart': #If message is multi part we only want the text version of the body, this walks the message and gets the body.
for part in msg.walk():
if part.get_content_type() == "text/plain":
body = part.get_payload(decode=True)
regex = re.compile(
"\\A.{0,}\\s") # First line of plain text body
matchArray = regex.findall(body)
body = str(matchArray[0])
body = ''.join(body.splitlines())
#notify
self.notify(
email.utils.parseaddr(msg['From'])[1], msg['Subject'][:64],
body[:64])
#add this message to the list of known messages
self.knownAboutMail.append(id)
debugMsg('showNewMailMessages() exited')
def get_latest_email_sent_to(self, email_address, timeout=300, poll=1):
start_time = time.time()
while ((time.time() - start_time) < timeout):
# It's no use continuing until we've successfully selected
# the inbox. And if we don't select it on each iteration
# before searching, we get intermittent failures.
status, data = self.imap.select('Inbox')
if status != 'OK':
time.sleep(poll)
continue
status, data = self.imap.search(None, 'TO', email_address)
data = [d for d in data if d is not None]
if status == 'OK' and data:
for num in reversed(data[0].split()):
status, data = self.imap.fetch(num, '(RFC822)')
email_msg = email.message_from_string(data[0][1])
return email_msg
time.sleep(poll)
raise AssertionError("No email sent to '%s' found in inbox "
"after polling for %s seconds." % (email_address, timeout))
def get_latest_email_sent_to(self, email_address, timeout=300, poll=1):
start_time = time.time()
while ((time.time() - start_time) < timeout):
# It's no use continuing until we've successfully selected
# the inbox. And if we don't select it on each iteration
# before searching, we get intermittent failures.
status, data = self.imap.select('Inbox')
if status != 'OK':
time.sleep(poll)
continue
status, data = self.imap.search(None, 'TO', email_address)
data = [d for d in data if d is not None]
if status == 'OK' and data:
for num in reversed(data[0].split()):
status, data = self.imap.fetch(num, '(RFC822)')
email_msg = email.message_from_string(data[0][1])
return email_msg
time.sleep(poll)
raise AssertionError("No email sent to '%s' found in inbox "
"after polling for %s seconds." % (email_address, timeout))
def showNewMailMessages(self):
debugMsg('showNewMailMessages() entered')
#get IDs of all UNSEEN messages
typ, data = self.imap.SEARCH(None, 'UNSEEN')
debugMsg('data - new mail IDs:')
debugMsg(data, 0)
for id in data[0].split():
if not id in self.knownAboutMail:
#get the entire message
etyp, edata = self.imap.FETCH(id, '(RFC822)')
raw= edata[0][1] #puts message from list into string
msg=email.message_from_string(raw)# converts string to instance of message msg is an email message so multipart and walk work on it.
body=''
#Finds the plain text version of the body of the message.
if msg.get_content_maintype() == 'multipart': #If message is multi part we only want the text version of the body, this walks the message and gets the body.
for part in msg.walk():
if part.get_content_type() == "text/plain":
body = part.get_payload(decode=True)
regex = re.compile("\\A.{0,}\\s") # First line of plain text body
matchArray = regex.findall(body)
body=str(matchArray[0])
body=''.join(body.splitlines())
#notify
self.notify(email.utils.parseaddr(msg['From'])[1], msg['Subject'][:64], body[:64] )
#add this message to the list of known messages
self.knownAboutMail.append(id)
debugMsg('showNewMailMessages() exited')
# Searches for emails based on subject
found = {}
y = 0
for subject in SUBJECTS:
if connection.search(None, '(SUBJECT "%s")' % (subject)):
status, num = connection.search(None, '(SUBJECT "%s")' % (subject))
split = num[0].split()
for item in split:
found[item] = subject
# Assembles raw email data into human readable message
y = 0
for key in found:
status, msg_data = connection.fetch(key, '(RFC822)')
mail = email.message_from_string(msg_data[0][1])
for part in mail.walk():
if part.get_filename() != None:
fileName = part.get_filename()
filePath = os.path.join(QUEUE, fileName)
if bool(fileName):
if os.path.isfile(filePath):
newName = str(y) + fileName
newPath = os.path.join(QUEUE, newName)
fp = open(newPath, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
y += 1
else:
fp = open(filePath, 'wb')
fp.write(part.get_payload(decode=True))
def fetch_message(self, num):
self.imap.select('Inbox')
status, data = self.imap.fetch(str(num), '(RFC822)')
email_msg = email.message_from_string(data[0][1])
return email_msg
def retrieveMail(self):
config = Config("./mail2Shell.ini")
tmp = config.trustedSender
trustedSender = tmp.split(";")
if config.enableGPG:
gpgProperlyConfigured = False
fingerprints = {}
keyDir = config.keys
gpgPath = os.path.join(keyDir, "gpgHome")
os.system('rm -rf ' + gpgPath)
#TODO REMOVE binary or make it co nfig
gpg = gnupg.GPG(homedir=gpgPath,
binary=config.gpgBinary,
options='')
if (os.path.exists(keyDir)):
for s in trustedSender:
fName = os.path.join(keyDir, s + ".asc")
if not (os.path.exists(fName)):
pass
#print "WARN: cannot handle gpg message for %s" % (s)
else:
f = open(fName)
tmp = f.read()
f.close()
res = gpg.import_keys(tmp)
fingerprints[s.upper()] = res.fingerprints[0]
fName = os.path.join(keyDir, config.userName + ".asc")
if not (os.path.exists(fName)):
self.log.error(
"gpg will not work as there is no key information for the main user %s"
% (config.userName))
else:
f = open(fName)
tmp = f.read()
f.close()
impRes = gpg.import_keys(tmp)
fingerprints[
config.userName.upper()] = impRes.fingerprints[0]
isPrivateKey = False
for x in impRes.results:
if x['status'] == 'Contains private key\n':
isPrivateKey = True
break
if not (isPrivateKey):
self.log.error("Key for %s contains no private key" %
(config.userName))
else:
gpgProperlyConfigured = True
db = config.hashStore
if os.path.exists(db) and os.path.isfile(
db) and os.stat(db).st_size > 0:
with open(db) as fp:
hashes = json.load(fp, cls=DateTimeDecoder)
else:
hashes = {}
else:
self.log.error("key Directory is missing (%s) " % (keyDir))
if config.allowOnlyGPG and not (gpgProperlyConfigured):
self.log.error(
"GPG not configured properly, but set up to work w/ encrypted messages only - exiting"
)
return
mail = imaplib.IMAP4_SSL(config.mailServer)
mail.login(config.userName, config.password)
lst = mail.list()
mail.select("inbox")
result, data = mail.uid('search', None, '(UNSEEN SUBJECT "' +
config.magicKey + '")') # (ALL/UNSEEN)
i = len(data[0].split())
for x in range(i):
latest_email_uid = data[0].split()[x]
#print latest_email_uid
result, email_data = mail.uid('fetch', latest_email_uid,
'(RFC822)')
# result, email_data = conn.store(num,'-FLAGS','\\Seen')
# this might work to set flag to seen, if it doesn't already
raw_email = email_data[0][1]
raw_email_string = raw_email.decode('utf-8')
email_message = email.message_from_string(raw_email_string)
# Header Details
date_tuple = email.Utils.parsedate_tz(email_message['Date'])
if date_tuple:
local_date = datetime.datetime.fromtimestamp(
email.utils.mktime_tz(date_tuple))
local_message_date = "%s" % (str(
local_date.strftime("%a, %d %b %Y %H:%M:%S")))
emailFrom = str(
email.Header.make_header(
email.Header.decode_header(email_message['From'])))
email_to = str(
email.Header.make_header(
email.Header.decode_header(email_message['To'])))
subject = str(
email.Header.make_header(
email.Header.decode_header(email_message['Subject'])))
found = False
for ts in trustedSender:
if ts.upper() in emailFrom.upper():
found = True
p = re.compile('[^\s@<>]+@[^\s@<>]+\.[^\s@<>]+',
re.IGNORECASE)
returnMail = re.findall(p, ts)[0]
break
if not (found):
#print emailFrom
continue
self.log.debug("Received email from %s" % (returnMail))
# Body details
hasErr = False
errMessage = ""
attachments = []
attachmentsIn = {}
isEncryptedMail = False
done = False
while not (done) and not (hasErr):
for part in email_message.walk():
#print part.get_content_type()
if part.get_content_type() == "multipart/encrypted":
if not (returnMail.upper() in fingerprints):
errMessage = "Encrypted mail cannot be processed as key information is missing for %s" % (
returnMail)
self.log.error(errMessage)
hasErr = True
break
if not (config.userName.upper() in fingerprints):
errMessage = "Encrypted mail cannot be processed as key information is missing for %s" % (
config.userName)
self.log.error(errMessage)
hasErr = True
break
isEncryptedMail = True
if part.get_content_type() == "text/plain":
body = part.get_payload(decode=True).decode('utf-8')
done = True
elif part.get_content_maintype(
) != 'multipart' and part.get(
'Content-Disposition') is not None:
if isEncryptedMail:
if part.get_content_type(
) == "application/octet-stream":
encMsg = part.get_payload(decode=True)
#print len(encMsg)
hsh = base64.b64encode(
hmac.new(
"1234567890",
msg=encMsg,
digestmod=hashlib.sha256).digest())
if hsh in hashes.keys():
errMessage = "Saw same message again (hash %s)" % (
hsh)
self.log.error(errMessage)
hasErr = True
break
else:
now = datetime.datetime.now()
hashes[hsh] = now
while len(hashes) > config.hashBufferSize:
oldestHash = ""
oldest = now
for k in hashes.keys:
if hashes[k] < oldest:
oldestHash = k
oldest = hashes[k]
if len(oldestHash) > 0:
del oldestHash[hashes]
with open(db + "_tmp", 'w') as fp:
json.dump(hashes,
fp,
cls=DateTimeEncoder)
os.rename(db + "_tmp", db)
decMsg = gpg.decrypt(encMsg)
if decMsg.ok:
if decMsg.signature_id is None:
print "Message is not signed"
if config.verifySignature:
self.log.error(
"email is not signed (missing)"
)
hasErr = True
errMessage = "Missing signature"
else:
print "Message is not signed"
else:
if fingerprints[returnMail.upper(
)] == decMsg.fingerprint:
self.log.debug(
"Message is signed by " +
returnMail)
else:
if config.verifySignature:
self.log.error(
"ERROR: email is not signed by trusted sender (fingerprint not matching)"
)
hasErr = True
errMessage = "Wrong signature"
else:
self.log.warn(
"email is not signed by trusted sender"
)
#print decMsg
#TODO check error message
email_message = email.message_from_string(
decMsg.data)
else:
hasErr = True
errMessage = decMsg.stderr
break
else:
attachmentsIn[part.get_filename(
)] = part.get_payload(decode=True)
else:
continue
if not (hasErr):
lines = body.split('\n')
retDataPlain = ""
retDataHtml = "<br>"
nextTo = 1.0
process = subprocess.Popen(config.path2Shell,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
for l in lines:
l = l.strip()
if len(l) == 0:
continue
if l.startswith('++'):
break
if l.startswith('+'):
if l.upper().startswith("+GET"):
fls = l[4:].split()
for fl in fls:
if os.path.exists(fl) and os.path.isfile(fl):
bytes_read = open(fl, "rb").read()
att = MIMEApplication(bytes_read,
Name=basename(fl))
att['Content-Disposition'] = 'attachment; filename="%s"' % basename(
fl)
attachments.append(att)
retDataPlain += "## Attached file " + fl + "\n"
retDataHtml += "## Attached file " + fl + "<br>"
else:
retDataPlain += "## Requested file " + fl + " does not exist or is not a file\n"
retDataHtml += "## Requested file " + fl + " does not exist or is not a file<br>"
if l.upper().startswith("+PUT"):
fls = l[4:].split()
for fl in fls:
if attachmentsIn.has_key(basename(fl)):
fp = open(fl, 'wb')
fp.write(attachmentsIn[basename(fl)])
fp.close()
retDataPlain += "## Stored file " + fl + "\n"
retDataHtml += "## Stored file " + fl + "<br>"
else:
retDataPlain += "## Attachment not found " + fl + "\n"
retDataHtml += "## Attachment not found " + fl + "<br>"
if l.upper().startswith('+TO'):
tmp = l[3:0].split()
if len(tmp) > 0:
try:
nextTo = float(tmp[0])
except ValueError:
nextTo = 1
self.retMessage(
"## Timeout parameter not recognized, assumung 1 second",
[retDataPlain, retDataHtml])
else:
self.retMessage(
"## Timeout command requires a parameter (float number)",
[retDataPlain, retDataHtml])
else:
retDataPlain += returnMail + "> " + l + "\n"
retDataHtml += returnMail + ">" + l + "<br>"
process.stdin.write(l + '\n')
fds = select.select([process.stdout, process.stderr],
[], [], nextTo)
nextTo = 1.0
outData = ""
for f in fds[0]:
fl = fcntl.fcntl(f, fcntl.F_GETFL)
fcntl.fcntl(f, fcntl.F_SETFL, fl | os.O_NONBLOCK)
outData += f.read()
retDataPlain += outData + "\n"
retDataHtml += outData.replace('\n',
'<br>') + "<br>"
process.communicate()
else:
retDataPlain = "Error occured: \n" + errMessage
retDataHtml = "Error occured: \n" + errMessage.replace(
'\n', '<br>')
smtp = smtplib.SMTP_SSL(config.smtpServer)
smtp.set_debuglevel(0)
smtp.login(config.userName, config.password)
from_addr = config.userName
to_addr = returnMail
msg = MIMEMultipart('alternative')
htmlData = "<html><head></head><body style=\"background:#6E6E6E\"><font face=\"Courier New\"><span style=\"color:#16CC13\"/>" \
+ retDataHtml + "</p></font></body></html>"
part1 = MIMEText(retDataPlain, "plain")
part2 = MIMEText(htmlData, "html")
if len(attachments) == 0:
msg.attach(part1)
msg.attach(part2)
msgToSend = msg
else:
msgMixed = MIMEMultipart('mixed')
msg.attach(part1)
msg.attach(part2)
msgMixed.attach(msg)
for att in attachments:
msgMixed.attach(att)
msgToSend = msgMixed
msgToSend['Subject'] = config.magicKey
msgToSend['From'] = config.userName
msgToSend['To'] = returnMail
if isEncryptedMail:
self.log.debug("Preparing encrypted mail")
encMsg = gpg.encrypt(
msgToSend.as_string(),
fingerprints[returnMail.upper()],
always_trust=True,
default_key=fingerprints[config.userName.upper()])
msg = MIMEMultipart('encrypted')
part1 = MIMEText('Version: 1\n', 'application/pgp-encrypted')
att = MIMEApplication(encMsg.data, Name='encrypted.asc')
att['Content-Disposition'] = 'inline; filename="encrypted.asc"'
att['Content-Description'] = 'OpenPGP encrypted message'
msg.attach(part1)
msg.attach(att)
msgToSend = msg
msgToSend['From'] = config.userName
msgToSend['To'] = returnMail
msg['Subject'] = config.magicKey
self.log.debug("Sending mail to %s" % (to_addr))
smtp.sendmail(from_addr, to_addr, msgToSend.as_string())
smtp.quit()
def fetch_message(self, num):
self.imap.select('Inbox')
status, data = self.imap.fetch(str(num), '(RFC822)')
email_msg = email.message_from_string(data[0][1])
return email_msg
# Searches for emails based on subject
found = {}
y = 0
for subject in SUBJECTS:
if connection.search(None, '(SUBJECT "%s")' % (subject)):
status, num = connection.search(None, '(SUBJECT "%s")' % (subject))
split = num[0].split()
for item in split:
found[item] = subject
# Assembles raw email data into human readable message
y = 0
for key in found:
status, msg_data = connection.fetch(key, '(RFC822)')
mail = email.message_from_string(msg_data[0][1])
for part in mail.walk():
if part.get_filename() != None:
fileName = part.get_filename()
filePath = os.path.join(QUEUE, fileName)
if bool(fileName):
if os.path.isfile(filePath):
newName = str(y) + fileName
newPath = os.path.join(QUEUE, newName)
fp = open(newPath, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
y += 1
else:
fp = open(filePath, 'wb')
fp.write(part.get_payload(decode=True))