class EmailOperation(object):
def __init__(self, user_name, pwd):
print("使用imapclient，操作邮件...")
self.server = IMAPClient('imap.139.com', use_uid=True, ssl=False)
self.username = user_name
self.password = pwd
def logout(self):
self.server.logout()
# 删除邮件
def _del(self, messages):
# print('del message')
self.server.delete_messages(messages)
self.server.expunge()
def _mv(self, message, folder):
self.server.copy(message, folder)
time.sleep(1)
self._del(message)
def _into(self, box):
# 'INBOX','草稿箱','已发送','已删除','100'
self.server.select_folder(box)
def _get_uids(self):
# 获取序列
uids = self.server.search(['NOT', 'DELETED'])
return uids
def email_body(self, mss):
# 获取某序列id的原始信息
raw_messages = self.server.fetch([mss], ['BODY[]', 'FLAGS'])
# pprint.pprint(rawMessages)
# 使用pyzmail，返回主体信息
message = pyzmail.PyzMessage.factory(raw_messages[mss][b'BODY[]'])
# 如果信息为空，返回None
if not message:
return None
# 获取正文内容
if message.text_part != None:
lines = message.text_part.get_payload().decode(
message.text_part.charset)
str = ''
# 去除内容中的回车
for line in lines:
if line not in ['\n', '\r']:
str += line
print(str)
# body自动
body = {
'subject': message.get_subject(),
'from': message.get_address('from'),
'to': message.get_address('to'),
'mainbody': str
}
return body
def del_all_message(self, folder):
'''清空某个文件夹下的所有邮件'''
self._into(folder)
uids = self._get_uids()
if len(uids) == 0:
return
else:
self._del(uids)
def del_new_message(self, fforlder):
'''把最近一封新邮件已到目的目录'''
self._into(fforlder)
uids = self._get_uids()
if len(uids) == 0:
return
else:
self._del(uids[-1])
def move_all_message_to_folder(self, fforlder, tforlder):
'''把maessage/messages 从 ffolder 移动到 tforlder'''
self._into(fforlder)
uids = self._get_uids()
# 邮件数量为空
if len(uids) == 0:
print("%s 数量为：0，该操作无效" % fforlder)
return True
self._mv(uids, tforlder)
def check_new_message(self):
'''判断最新一封邮件，是否包含某个字段，显示邮件数量'''
self._into("INBOX")
uids = self._get_uids()
print('current INBOX email: %s' % str(len(uids)))
if len(uids) in [0, 1]:
print("不执行，目前邮件数量为：%r" % len(uids))
return
if self.email_body(uids[-1])['subject'] == 'testReceive':
self._del(uids[-1])
def delete_newest_mail(self):
'''删除最新的一封邮件'''
try:
is_true = False
self.server.login(self.username, self.password)
self.del_new_message('INBOX')
is_true = True
except BaseException as error:
print(error)
print("删除邮件可能出现错误")
finally:
self.logout()
return is_true
def clear_forlder(self, l=[]):
'''清空邮箱某个文件夹'''
'''
sample:
clearForlder(['100', 'INBOX'])
'''
is_true = False
if len(l) == 0:
return is_true
try:
self.server.login(self.username, self.password)
for f in l:
print("clear Forlder: %s" % f)
self.del_all_message(f)
time.sleep(1)
is_true = True
except BaseException as error:
print(error)
print("删除邮件可能出现错误")
finally:
self.logout()
return is_true
def move_forlder(self, l=[]):
'''移动邮件
sample:
moveForlder(['100', 'INBOX'])
'''
is_true = False
if len(l) == 0:
return is_true
try:
self.server.login(self.username, self.password)
self.move_all_message_to_folder(l[0], l[1])
print("移动邮件成功：%s => %s" % (l[0], l[1]))
is_true = True
except BaseException as error:
print(error)
print("清空邮箱某个文件夹可能出现错误")
finally:
self.logout()
return is_true
def check_inbox_cnt(self):
'''获取邮件数量'''
try:
is_true = 0
self.server.login(self.username, self.password)
self._into("INBOX")
uids = self._get_uids()
# 数量为 0
if len(uids) == 0:
return 0
# 判断
if len(uids) == 100:
print("100封邮件")
return 0
elif len(uids) < 100:
print('邮件数量少于100封')
return 0
else:
cnt = len(uids) - 100
print('需要删除邮件数量为：%d' % cnt)
is_true = cnt
except BaseException as error:
print(error)
print("删除邮件可能出现错误")
finally:
self.logout()
return is_true
def check_inbox(self):
'''确保收件箱有100封邮件'''
try:
is_true = True
self.server.login(self.username, self.password)
self._into("INBOX")
uids = self._get_uids()
all = len(uids)
# 数量为 0
if all == 0:
return False
# 判断
if all == 100:
print("100封邮件")
return is_true
elif all < 100:
print('邮件数量少于100封')
return False
else:
print('需要删除邮件数量为：%d' % (all - 100))
# print(Uids[100:])
self._del(uids[100:])
return is_true
except BaseException as error:
print(error)
print("删除邮件可能出现错误")
is_true = False
finally:
self.logout()
return is_true
def seen(self):
'''将收件箱邮件，标记已读'''
self.server.login(self.username, self.password)
self._into("INBOX")
# 搜索 未读邮件
typ = self.server.search([u'UNSEEN'])
# 把邮件改为已读
for num in typ:
print(num)
self.server.set_flags(num, [u'Seen'])
class EmailClient(object):
def __init__(self, config):
"""
Create an email client abstraction configured for an account.
config is a dict with keys:
HOST
USERNAME
PASSWORD
USE_SSL default=True
FOLDER default='INBOX'
SELECTORS default=['NOT DELETED']
"""
self.config = {
'USE_SLL': True,
'FOLDER': 'INBOX',
'SELECTORS': ['NOT DELETED']
}
self.config.update(config)
def connect(self):
self.server = IMAPClient(self.config['HOST'],
use_uid=True,
ssl=self.config['USE_SSL'])
self.server.login(self.config['USERNAME'], self.config['PASSWORD'])
select_info = self.server.select_folder(self.config['FOLDER'])
def process_new_messages(self, debug=False):
"""
Get mesages from the server.
Note: new information on existing uids will be ignored.
For example, if the rfc_size changes (which is a strangely common occurrence),
the new value will be ignored.
"""
if debug: print "searching"
messages = self.server.search(self.config['SELECTORS'])
if debug: print "done searching"
self._process_messages(messages, debug=debug)
def _process_messages(self, uids, move=False, debug=False):
if debug: print "fetching"
response = self.server.fetch(uids, ['FLAGS', 'RFC822', 'RFC822.SIZE'])
for msg_uid, data in response.iteritems():
if debug: print "processing %s" % msg_uid
with transaction.commit_on_success():
# extract data
flags = data['FLAGS']
rfc_size = data['RFC822.SIZE']
raw_body = data['RFC822']
seq = data['SEQ']
# save objects
email, _ = EmailMessage.objects.get_or_create(
uid=msg_uid,
username=self.config['USERNAME'],
host=self.config['HOST'],
defaults={
'raw_body': raw_body,
'rfc_size': rfc_size,
'seq': data['SEQ'],
})
email.save()
for flag in flags:
EmailFlag.objects.get_or_create(email=email,
flag=flag)[0].save()
# move message
if move:
self.server.copy([msg_uid], 'auto_processed')
self.server.delete_messages([msg_uid])
class IMAP():
"""
Central class for IMAP server communication
"""
Retval = namedtuple('Retval', 'code data')
def __init__(self, logger, username, password,
server='localhost',
port=143,
starttls=False,
imaps=False,
tlsverify=True,
test=False,
timeout=None):
self.logger = logger
self.username = username
self.password = password
self.server = server
self.port = port
self.imaps = imaps
self.starttls = starttls
self.timeout = timeout
self.sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) # TODO add proto arg
if tlsverify:
self.sslcontext.verify_mode = ssl.CERT_REQUIRED
else:
self.sslcontext.verify_mode = ssl.CERT_NONE # TODO improve?
self.test = test
self.conn = None
def do_select_mailbox(func):
"""
Decorator to do a fresh mailbox SELECT
"""
def wrapper(*args, **kwargs):
if len(args) != 1:
raise AttributeError(
'Size of *args tuple "{0}" isn\'t 1. It looks like you haven\'t specified all '
'method arguments as named arguments!'.format(
args))
mailbox = None
for key in ['mailbox', 'source']:
if key in kwargs.keys():
mailbox = kwargs[key]
break
if mailbox is None:
raise KeyError('Unable to SELECT a mailbox, kwargs "{0}" doesn\'t contain a mailbox name'.format(kwargs))
result = args[0].select_mailbox(mailbox)
if not result.code:
raise RuntimeError(result.data)
return func(*args, **kwargs)
return wrapper
def process_error(self, exception, simple_return=False):
"""
Process Python exception by logging a message and optionally showing traceback
"""
trace_info = exc_info()
err_msg = str(exception)
if isinstance(exception, IMAPClient.Error):
err_msg = Helper().byte_to_str(exception.args[0])
self.logger.error('Catching IMAP exception %s: %s', type(exception), err_msg)
if self.logger.isEnabledFor(loglevel_DEBUG):
print_exception(*trace_info)
del trace_info
if simple_return:
return exception
else:
return self.Retval(False, err_msg)
def connect(self, retry=True, logout=False):
"""
Connect to IMAP server and login
"""
if self.starttls:
self.logger.debug('Establishing IMAP connection using STARTTLS/%s to %s and logging in with user %s', self.port, self.server,
self.username)
elif self.imaps:
self.logger.debug('Establishing IMAP connection using SSL/%s (imaps) to %s and logging in with user %s', self.port, self.server,
self.username)
login = ''
err_return = None
try:
self.conn = IMAPClient(host=self.server,
port=self.port,
use_uid=True,
ssl=self.imaps,
ssl_context=self.sslcontext,
timeout=self.timeout)
if self.starttls:
self.conn.starttls(ssl_context=self.sslcontext)
login = self.conn.login(self.username, self.password)
login_response = Helper().byte_to_str(login)
# Test login/auth status
login_success = False
noop = self.noop()
if noop.code and noop.data:
login_success = True
if logout:
return self.disconnect()
elif login_success:
return self.Retval(True, login_response)
else:
return self.Retval(False, login_response) # pragma: no cover
except Exception as e:
err_return = self.process_error(e)
if err_return.data == '[AUTHENTICATIONFAILED] Authentication failed.':
return err_return
if retry:
self.logger.error('Trying one more time to login')
sleep(2)
return self.connect(retry=False, logout=logout)
return err_return
def noop(self):
"""
Do a noop to test login status
"""
try:
noop = self.conn.noop()
noop_response = Helper().byte_to_str(noop[0])
noop_resp_pattern_re = regex_compile('^(Success|NOOP completed)')
login_success = noop_resp_pattern_re.match(noop_response)
return self.Retval(True, login_success)
except IMAPClient.Error as e:
return self.process_error(e)
def disconnect(self):
"""
Disconnect from IMAP server
"""
result = self.conn.logout()
response = Helper().byte_to_str(result)
return self.Retval(response == 'Logging out', response)
def list_mailboxes(self, directory='', pattern='*'):
"""
Get a listing of folders (mailboxes) on the server
"""
try:
raw_list = self.conn.list_folders(directory, pattern)
nice_list = []
for mailbox in raw_list:
flags = []
for flag in mailbox[0]:
flags.append(flag.decode('utf-8'))
nice_list.append({'name': mailbox[2], 'flags': flags, 'delimiter': mailbox[1].decode("utf-8")})
return self.Retval(True, nice_list)
except IMAPClient.Error as e:
return self.process_error(e)
def select_mailbox(self, mailbox):
"""
Select a mailbox to work on
"""
self.logger.debug('Switching to mailbox %s', mailbox)
try:
result = self.conn.select_folder(mailbox)
response = {}
for key, value in result.items():
unicode_key = Helper().byte_to_str(key)
if unicode_key == 'FLAGS':
flags = []
for flag in value:
flags.append(Helper().byte_to_str(flag))
response[unicode_key] = tuple(flags)
else:
response[unicode_key] = value
return self.Retval(True, response)
except IMAPClient.Error as e:
return self.process_error(e)
def add_mail(self, mailbox, message, flags=(), msg_time=None):
"""
Add/append a mail to a mailbox
"""
self.logger.debug('Adding a mail into mailbox %s', mailbox)
try:
if not isinstance(message, Mail):
message = Mail(logger=self.logger, mail_native=message)
message_native = message.get_native()
#self.conn.append(mailbox, message, flags, msg_time)
self._append(mailbox, str(message_native), flags, msg_time)
# According to rfc4315 we must not return the UID from the response, so we are fetching it ourselves
uids = self.search_mails(mailbox=mailbox, criteria='HEADER Message-Id "{0}"'.format(message.get_header('Message-Id'))).data[0]
return self.Retval(True, uids)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def search_mails(self, mailbox, criteria='ALL', autocreate_mailbox=False):
"""
Search for mails in a mailbox
"""
self.logger.debug('Searching for mails in mailbox %s and criteria=\'%s\'', mailbox, criteria)
try:
return self.Retval(True, list(self.conn.search(criteria=criteria)))
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def fetch_mails(self, uids, mailbox, return_fields=None):
"""
Retrieve mails from a mailbox
"""
self.logger.debug('Fetching mails with uids %s', uids)
return_raw = True
if return_fields is None:
return_raw = False
return_fields = [b'RFC822']
mails = {}
try:
for uid in uids:
result = self.conn.fetch(uid, return_fields)
if not result:
continue
if return_raw:
mails[uid] = result[uid]
else:
#mails[uid] = Mail(logger=self.logger, uid=uid, mail_native=email.message_from_bytes(result[uid][b'RFC822']))
mails[uid] = Mail(logger=self.logger, mail_native=email.message_from_bytes(result[uid][b'RFC822']))
return self.Retval(True, mails)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def get_mailflags(self, uids, mailbox):
"""
Retrieve flags from mails
"""
try:
result = self.conn.get_flags(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error('Failed to get flags for mail with uid=%s: %s', uid, result)
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def set_mailflags(self, uids, mailbox, flags=[]):
"""
Set and retrieve flags from mails
"""
if self.test:
self.logger.info('Would have set mail flags on message uids "%s"', str(uids))
return self.Retval(True, None)
else:
self.logger.debug('Setting flags=%s on mails uid=%s', flags, uids)
try:
result = self.conn.set_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error('Failed to set and get flags for mail with uid=%s: %s', uid, result)
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def add_mailflags(self, uids, mailbox, flags=[]):
"""
Add and retrieve flags from mails
"""
if self.test:
self.logger.info('Would have added mail flags on message uids "%s"', str(uids))
return self.Retval(True, None)
else:
self.logger.debug('Adding flags=%s on mails uid=%s', flags, uids)
try:
result = self.conn.add_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error('Failed to add and get flags for mail with uid=%s: %s', uid, result)
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def move_mail(self, message_ids, source, destination, delete_old=True, expunge=True, add_flags=None, set_flags=None):
"""
Move a mail from a mailbox to another
"""
return self.copy_mails(message_ids=message_ids,
source=source,
destination=destination,
delete_old=delete_old,
expunge=expunge,
add_flags=add_flags,
set_flags=set_flags)
@do_select_mailbox
def copy_mails(self, source, destination, message_ids=None, delete_old=False, expunge=False, add_flags=None, set_flags=None):
"""
Copies one or more mails from a mailbox into another
"""
if self.test:
if delete_old:
self.logger.info('Would have moved mail Message-Ids="%s" from "%s" to "%s", skipping because of beeing in testmode',
message_ids, source, destination)
else:
self.logger.info('Would have copied mails with Message-Ids="%s" from "%s" to "%s", skipping because of beeing in testmode',
message_ids, source, destination)
return self.Retval(True, None)
else:
try:
if delete_old:
self.logger.debug('Moving mail Message-Ids="%s" from "%s" to "%s"', message_ids, source, destination)
else:
self.logger.debug('Copying mail Message-Ids="%s" from "%s" to "%s"', message_ids, source, destination)
#if message_ids is None:
# message_ids = []
# result = self.fetch_mails(uids=uids, mailbox=source)
# if not result.code:
# self.logger.error('Failed to determine Message-Id by uids for mail with uids "%s"', uids)
# return result
# message_ids.append(result.data.keys())
if not self.mailbox_exists(destination).data:
self.logger.info('Destination mailbox %s doesn\'t exist, creating it for you', destination)
result = self.create_mailbox(mailbox=destination)
if not result.code:
self.logger.error('Failed to create the mailbox %s: %s', source, result.data) # pragma: no cover
return result # pragma: no cover
uids = []
for message_id in message_ids:
result = self.search_mails(mailbox=source, criteria='HEADER Message-Id "{0}"'.format(message_id))
if not result.code or len(result.data) == 0:
self.logger.error('Failed to determine uid by Message-Id for mail with Message-Id "%s"', message_id)
return self.Retval(False, result.data)
uids.append(result.data[0])
result = self.select_mailbox(source)
if not result.code:
return result # pragma: no cover
self.conn.copy(uids, destination)
if delete_old:
result = self.delete_mails(uids=uids, mailbox=source)
if not result.code:
self.logger.error('Failed to remove old mail with Message-Id="%s"/uids="%s": %s', message_ids, uids,
result.data) # pragma: no cover
return result # pragma: no cover
if expunge: # TODO don't expunge by default
result = self.expunge(mailbox=source)
if not result.code:
self.logger.error('Failed to expunge on mailbox %s: %s', source, result.data) # pragma: no cover
return result # pragma: no cover
dest_uids = []
for message_id in message_ids:
result = self.search_mails(mailbox=destination, criteria='HEADER Message-Id "{0}"'.format(message_id))
if not result.code:
self.logger.error('Failed to determine uid by Message-Id for mail with Message-Id "%s"',
message_id) # pragma: no cover
return result # pragma: no cover
dest_uids.append(result.data[0])
if isinstance(set_flags, list):
self.set_mailflags(uids=dest_uids, mailbox=destination, flags=set_flags)
if add_flags:
self.add_mailflags(uids=dest_uids, mailbox=destination, flags=add_flags)
return self.Retval(True, dest_uids)
except IMAPClient.Error as e:
return self.process_error(e)
def _append(self, folder, msg, flags=(), msg_time=None): # TODO
"""
FORKED FORM IMAPCLIENT
"""
if msg_time:
if not msg_time.tzinfo: # pragma: no cover
msg_time = msg_time.replace(tzinfo=FixedOffset.for_system()) # pragma: no cover
time_val = '"{0}"'.format(msg_time.strftime("%d-%b-%Y %H:%M:%S %z"))
time_val = imapclient.imapclient.to_unicode(time_val)
else:
time_val = None
return self.conn._command_and_check('append', self.conn._normalise_folder(folder), imapclient.imapclient.seq_to_parenstr(flags),
time_val, Helper.str_to_bytes(msg),
unpack=True)
@do_select_mailbox
def expunge(self, mailbox):
"""
Expunge mails form a mailbox
"""
self.logger.debug('Expunge mails from mailbox %s', mailbox)
try:
return self.Retval(True, b'Expunge completed.' in self.conn.expunge())
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
def create_mailbox(self, mailbox):
"""
Create a mailbox
"""
self.logger.debug('Creating mailbox %s', mailbox)
try:
return self.Retval(True, self.conn.create_folder(mailbox) == b'Create completed.')
except IMAPClient.Error as e:
return self.process_error(e)
def mailbox_exists(self, mailbox):
"""
Check whether a mailbox exists
"""
try:
return self.Retval(True, self.conn.folder_exists(mailbox))
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
@do_select_mailbox
def delete_mails(self, uids, mailbox):
"""
Delete mails
"""
self.logger.debug('Deleting mails with uid="%s"', uids)
try:
result = self.conn.delete_messages(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error('Failed to get flags for mail with uid=%s after deleting it: %s', uid, result)
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
by_size = sorted(list(response.viewitems()), key=lambda m: m[1]['RFC822.SIZE'])
sizes = [ m[1]['RFC822.SIZE'] for m in by_size ]
# select msgs that are bigger than threshold
bigger = bisect.bisect_right(sizes, options.threshold) # index of the first message bigger than threshold
big_messages = by_size[bigger:]
print "There are %d messages bigger than %s" % (len(big_messages), options.threshold)
big_uids = [ msg[0] for msg in big_messages ]
def print_messages():
print "\n--- Messages bigger than {}: ".format(options.threshold).ljust(80, '-')
print "Format: <date> | <size> | <from> | <subject>"
for msgid, data in big_messages:
headers = email.message_from_string(data['RFC822.HEADER'])
size = data['RFC822.SIZE']
date = data['INTERNALDATE']
print "{} | {} | {} | {}".format(date, size, headers['from'], headers['subject'])
if not options.no_label:
print "Labeling big messages with label %s" % options.label
server.create_folder(options.label)
server.copy(big_uids, options.label)
print "Your messages larger than {} bytes have been labeled with label {}".format(options.threshold, options.label)
if options.print_msgs or options.no_label:
print_messages()
class SourceDriverMail(SourceDriver):
def __init__(self, instanceName: str, settings: Settings,
parser: MessageParser) -> None:
super().__init__("mail", instanceName, settings, parser)
# Settings
self.__server = self.getSettingString("server", "")
self.__user = self.getSettingString("user", "")
self.__password = self.getSettingString("password", "")
self.__ssl = self.getSettingBoolean("ssl", True)
self.__fix_weak_dh = self.getSettingBoolean("fix_weak_dh", False)
self.__allowlist = self.getSettingList("allowlist", [])
self.__denylist = self.getSettingList("denylist", [])
self.__cleanup = self.getSettingString("cleanup", "")
self.__archive_folder = self.getSettingString("archive_folder",
"Archive")
if self.__cleanup == "Delete":
self.print("Cleanup strategy is delete")
elif self.__cleanup == "Archive":
self.print("Cleanup strategy is archive")
elif self.__cleanup == "":
self.print("Cleanup is disabled")
else:
self.fatal("Unknown cleanup strategy")
# Internal
self.__healthy = False
self.__connect()
def retrieveEvent(self) -> Optional[SourceEvent]:
try:
if self.isDebug():
self.print("Checking for new mails")
messages = self.__imap_client.search('UNSEEN')
for uid, message_data in self.__imap_client.fetch(
messages, "RFC822").items():
message = email.message_from_bytes(message_data[b"RFC822"],
policy=policy.default)
sender = parseaddr(message.get("From"))[1]
sourceEvent = SourceEvent()
sourceEvent.source = SourceEvent.SOURCE_MAIL
sourceEvent.timestamp = datetime.datetime.strptime(
message.get('Date'), "%a, %d %b %Y %H:%M:%S %z").strftime(
SourceEvent.TIMESTAMP_FORMAT)
sourceEvent.sender = sender
sourceEvent.raw = message.get_body(('plain', )).get_content()
if self.isSenderAllowed(allowlist=self.__allowlist,
denylist=self.__denylist,
sender=sender):
parsedSourceEvent = self.parser.parseMessage(
sourceEvent, None) # type: ignore[union-attr]
self.__do_cleanup(uid)
return parsedSourceEvent
else:
self.error("Received unhandled message (ignored sender)")
return UnhandledEvent.fromSourceEvent(
sourceEvent, UnhandledEvent.CAUSE_IGNORED_SENDER)
except (timeout, OSError) as e:
self.error("Connection to mailserver timed out")
self.__healthy = False
self.__connect()
def getSourceState(self) -> SourceState:
if self.__healthy:
return SourceState.OK
return SourceState.ERROR
def __connect(self):
if self.__fix_weak_dh:
context = ssl.SSLContext(
ssl.PROTOCOL_TLSv1_2) # Workaround for weak dh key
context.set_ciphers('DEFAULT@SECLEVEL=1')
else:
context = ssl.SSLContext()
try:
if self.isDebug():
self.print("Connecting to server {}".format(self.__server))
self.__imap_client = IMAPClient(self.__server,
use_uid=True,
ssl=self.__ssl,
ssl_context=context,
timeout=1.0)
except gaierror:
self.error("Failed to connect to Mail Server")
except ssl.SSLError:
self.fatal("Failed to connect to Mail Server (TLS Error)")
else:
try:
if self.isDebug():
self.print("Login as user {}".format(self.__user))
self.__imap_client.login(self.__user, self.__password)
except LoginError:
self.error("Mail Server login failed")
else:
if self.isDebug():
self.print("Login successful")
self.__healthy = True
self.__imap_client.select_folder('INBOX', readonly=False)
def __create_imap_folder(self, folder):
if not self.__imap_client.folder_exists(folder):
self.print("Folder {} does not exist creating")
self.__imap_client.create_folder(folder)
def __do_cleanup(self, uid):
if self.__cleanup == "Archive":
self.__create_imap_folder(self.__archive_folder)
self.__imap_client.copy(uid, self.__archive_folder)
if self.__cleanup == "Delete" or self.__cleanup == "Archive":
self.__imap_client.delete_messages(uid)
self.__imap_client.expunge(uid)
class MailToolbox(SourceFactory):
def __init__(self,
hote_imap,
nom_utilisateur,
mot_de_passe,
dossier_cible='INBOX',
verify_peer=True,
use_secure_socket=True,
legacy_secure_protocol=False):
super().__init__('IMAPFactory via {}'.format(hote_imap))
self._ssl_context = SSLContext(
protocol=PROTOCOL_TLS) if use_secure_socket else None
self._use_secure_socket = use_secure_socket
if verify_peer is False and use_secure_socket is True:
# don't check if certificate hostname doesn't match target hostname
self._ssl_context.check_hostname = False
# don't check if the certificate is trusted by a certificate authority
self._ssl_context.verify_mode = CERT_NONE
if legacy_secure_protocol and use_secure_socket:
self._ssl_context.options = OP_ALL
self._hote_imap = Session.UNIVERSELLE.retranscrire(hote_imap)
self._client = IMAPClient(host=self._hote_imap,
port=993 if self._use_secure_socket else 143,
ssl=self._use_secure_socket,
ssl_context=self._ssl_context)
self._nom_utilisateur = Session.UNIVERSELLE.retranscrire(
nom_utilisateur)
self._verify_peer = verify_peer
self._dossier_cible = dossier_cible
self._mot_de_passe = mot_de_passe
self._client.login(
self._nom_utilisateur,
Session.UNIVERSELLE.retranscrire(self._mot_de_passe))
self._echec = False
MailToolbox.INSTANCES.append(self)
@property
def est_hors_service(self):
return self._echec
def reset(self):
try:
self._client.logout()
except IMAPClientError as e:
pass
except OSError as e:
pass
try:
self._client = IMAPClient(
host=self._hote_imap,
port=993 if self._use_secure_socket else 143,
ssl=self._use_secure_socket,
ssl_context=self._ssl_context)
self._client.login(
self._nom_utilisateur,
Session.UNIVERSELLE.retranscrire(self._mot_de_passe))
except IMAPClientError as e:
logger.error(
"Une erreur IMAP critique est survenue lors de la reconnexion. {msg_err}",
msg_err=str(e))
self._echec = True
return
self._echec = False
@property
def dossier_cible(self):
return self._dossier_cible
@dossier_cible.setter
def dossier_cible(self, nouveau_dossier_cible):
if isinstance(nouveau_dossier_cible, str):
self._dossier_cible = nouveau_dossier_cible
@property
def hote_imap(self):
return self._hote_imap
@property
def nom_utilisateur(self):
return self._nom_utilisateur
@staticmethod
def fetch_instance(hote_imap, nom_utilisateur):
hote_imap, nom_utilisateur = Session.UNIVERSELLE.retranscrire(
hote_imap), Session.UNIVERSELLE.retranscrire(nom_utilisateur)
for inst in MailToolbox.INSTANCES:
if isinstance(
inst, MailToolbox
) is True and inst.nom_utilisateur == nom_utilisateur and inst.hote_imap == hote_imap:
return inst
return None
def extraire(self, no_progress_bar=True):
if self.est_hors_service is True:
self.reset()
try:
self._client.select_folder(self._dossier_cible)
except IMAPClientError as e:
raise ExtractionSourceException('IMAPClientError: ' + str(e))
except IMAPClientAbortError as e:
raise ExtractionSourceException('IMAPClientAbortError: ' + str(e))
except IMAPClientReadOnlyError as e:
raise ExtractionSourceException('IMAPClientReadOnlyError: ' +
str(e))
finally:
self._echec = True
# fetch selectors are passed as a simple list of strings.
responses = self._client.fetch(self._client.search(['NOT', 'DELETED']),
['UID', 'ENVELOPE', 'BODY', 'RFC822'])
extractions = list() # type: list[Mail]
for id_response in tqdm(responses.keys(
), unit=' message') if no_progress_bar is False else responses.keys():
email_message = email.message_from_bytes(
responses[id_response][b'RFC822'])
mail = Mail.from_message(email_message)
mail.folder = self._dossier_cible
mail.flags = responses[id_response][
b'FLAGS'] if b'FLAGS' in responses[id_response].keys(
) else tuple()
mail.bal_internal_id = id_response
extractions.append(mail)
mail.factory = self
return sorted(extractions,
key=lambda x: x.date_received or datetime.now())
def copier(self, mail, dossier_dest):
"""
:param Mail mail:
:param str dossier_dest:
:return:
"""
if self.est_hors_service is True:
self.reset()
try:
if self._client.folder_exists(dossier_dest) is False:
raise FileNotFoundError(
'Le dossier "{}" n\'existe pas sur le serveur IMAP distant !'
)
self._client.select_folder(mail.folder)
self._client.copy([mail.bal_internal_id], dossier_dest)
except IMAPClientError as e:
raise ManipulationSourceException('IMAPClientError: ' + str(e))
except IMAPClientAbortError as e:
raise ManipulationSourceException('IMAPClientAbortError: ' +
str(e))
except IMAPClientReadOnlyError as e:
raise ManipulationSourceException('IMAPClientReadOnlyError: ' +
str(e))
finally:
self._echec = True
def deplacer(self, mail, dossier_dest):
if self.est_hors_service is True:
self.reset()
try:
if self._client.folder_exists(dossier_dest) is False:
raise FileNotFoundError(
'Le dossier "{}" n\'existe pas sur le serveur IMAP distant !'
)
self._client.select_folder(mail.folder)
try:
self._client.move([mail.bal_internal_id], dossier_dest)
except CapabilityError as e:
self.copier(mail, dossier_dest)
self.supprimer(mail)
except IMAPClientError as e:
raise ManipulationSourceException('IMAPClientError: ' + str(e))
except IMAPClientAbortError as e:
raise ManipulationSourceException('IMAPClientAbortError: ' +
str(e))
except IMAPClientReadOnlyError as e:
raise ManipulationSourceException('IMAPClientReadOnlyError: ' +
str(e))
finally:
self._echec = True
def supprimer(self, mail):
if self.est_hors_service is True:
self.reset()
try:
self._client.select_folder(mail.folder)
self._client.delete_messages([mail.bal_internal_id], silent=True)
self._client.expunge([mail.bal_internal_id])
except IMAPClientError as e:
raise ManipulationSourceException('IMAPClientError: ' + str(e))
except IMAPClientAbortError as e:
raise ManipulationSourceException('IMAPClientAbortError: ' +
str(e))
except IMAPClientReadOnlyError as e:
raise ManipulationSourceException('IMAPClientReadOnlyError: ' +
str(e))
finally:
self._echec = True
class Miner:
"""create a new email miner instance that can be used to traverse mails"""
imap: IMAPClient = None
def __init__(self,
hostname: str,
username: str,
password: str,
port: int = imaplib.IMAP4_SSL_PORT,
use_ssl: bool = True,
verify: bool = True,
log_level: int = None):
"""
Create a new instance of the miner.
:param hostname: the hostname of the imap server to connect to
:param username: the user to login as
:param password:
:param port: the port to connect to. (defaults to 993)
:param use_ssl: whether to use SSL to connect (defaults to True)
:param verify: whether to verify the SSL certificates (defaults to False)
"""
if log_level is not None:
logging.basicConfig(
format='%(asctime)s - %(levelname)s: %(message)s',
level=log_level)
ssl_context = ssl.create_default_context()
if not verify:
# disable hostname check. certificate may not match hostname.
ssl_context.check_hostname = False
# disable certificate authority verification. certificate maybe issued by unknown CA
ssl_context.verify_mode = ssl.CERT_NONE
self.imap = IMAPClient(host=hostname,
port=port,
ssl=use_ssl,
ssl_context=ssl_context)
self.imap.login(username, password)
@contextlib.contextmanager
def folder(self, folder_name: str, read_only: bool = True):
"""
Switch to a specific folder.
:param folder_name: name of the folder to switch to
:param read_only: read-only mode will not mark emails as read even after retrieval
:return:
"""
try:
yield self.imap.select_folder(folder_name, read_only)
finally:
self.imap.close_folder()
@contextlib.contextmanager
def inbox(self, read_only: bool = True):
"""
Switch to the inbox folder.
:param read_only: read-only mode will not mark emails as read even after retrieval
:return:
"""
try:
yield self.imap.select_folder('inbox', read_only)
finally:
self.imap.close_folder()
def mark_as_unread(self, message_ids: List[int]):
"""
Mark the given message IDs as unread by removing the SEEN flag.
:param message_ids:
:return:
"""
self.imap.remove_flags(message_ids, [SEEN])
def mark_as_read(self, message_ids: List[int]):
"""
Mark the given message IDs as read by adding the SEEN flag.
:param message_ids:
:return:
"""
self.imap.add_flags(message_ids, [SEEN])
def delete(self, message_ids: List[int]):
"""
Delete the given message IDs
:param message_ids:
:return:
"""
self.imap.delete_messages(message_ids, True)
def archive(self, message_ids: List[int]):
"""
Archive the given message IDs
:param message_ids:
:return:
"""
self.imap.copy(message_ids, br'\Archive')
self.delete(message_ids)
def get_emails(self,
unread_only: bool = True,
with_body: bool = False,
keep_as_unread: bool = False,
in_memory: bool = True) -> List[Email]:
"""
Get emails from the selected folder.
:param keep_as_unread: keep any retrieved emails as unread in the mailbox.
:param unread_only: choose only to retrieve unread mails
:param with_body: read-only mode will not mark emails as read even after retrieval
:param in_memory: store the parsed attachments in-memory as bytes or to a temp file locally
:return:
"""
ids = self.imap.search('(UNSEEN)' if unread_only else 'ALL')
flags = ['ENVELOPE', 'FLAGS', 'UID', 'INTERNALDATE']
if with_body:
flags.append('BODY[]')
response = self.imap.fetch(ids, flags)
try:
if keep_as_unread:
self.mark_as_unread(ids)
else:
self.mark_as_read(ids)
except Exception:
# will throw an exception if folder in read-only mode. so ignore.
pass
return parse_emails(response, in_memory)
def __enter__(self):
"""
return the instance of the miner for use as a context manager.
:return:
"""
return self
def __exit__(self, *args):
"""
Close folder and logout on exit when used as a context manager.
:param args:
:return:
"""
if self.imap is not None:
try:
self.imap.close_folder()
except:
pass
self.imap.logout()
class Mail:
"""
Wrapper for all needed methods to process the mails on the IMAP-server.
Attributes
----------
config : config.Config
Configuration manager wrapping around the yaml-config-file.
"""
# List of all Content-types of attachments that should be saved seperatly
valid_ctypes = ['application/pdf']
# Regexp that matches all allowed characters in a filename
filesafe = re.compile(r'[\w\-\. ]')
def __init__(self, config):
config.checkParams('server', 'port', 'username', 'password', 'inbox',
'outbox', 'basepath', 'eml-to-pdf-path')
host = config.get('server')
port = config.get('port')
username = config.get('username')
password = config.get('password')
inbox = config.get('inbox')
self.outbox = config.get('outbox')
self.basepath = config.get('basepath')
if not os.path.exists(self.basepath): os.makedirs(self.basepath)
self.emltopdf = config.get('eml-to-pdf-path')
self.server = IMAPClient(host, port=port)
result = self.server.login(username, password)
print(result.decode('UTF-8'))
self.server.select_folder(inbox)
def check(self):
"""
Checks for new mail to process on the server.
"""
messages = self.server.search(['ALL'])
if len(messages) < 1: return
for id, data in self.server.fetch(messages, 'RFC822').items():
mail = email.message_from_bytes(data[b'RFC822'])
self.processMail(mail)
self.server.copy(messages, self.outbox)
self.server.delete_messages(messages)
self.server.expunge()
def getDate(self, mail):
"""
Extracts the date from the mail header and rewrites it for use as the
beginning of a filename.
Parameters
----------
mail : email
Mail from which to extract the date.
Returns
-------
The date reformatted to be used in a filename.
"""
date = mail.get('Date')
date = datetime.strptime(date[:-6], '%a, %d %b %Y %H:%M:%S')
return date.strftime('%Y-%m-%d-%H-%M-%S')
def processMail(self, mail):
"""
Processes a specific mail by saving it.
Parameters
----------
mail : email
The mail to process.
"""
filename = '%s-%s.eml' % (self.getDate(mail), mail.get('Subject'))
filename = self.validateFilename(filename)
filename = os.path.join(self.basepath, filename)
with open(filename, 'w') as f:
generator = email.generator.Generator(f)
generator.flatten(mail)
if mail.is_multipart(): self.processAttachments(mail)
def processAttachments(self, mail):
"""
Processes all attachments of a mail.
Parameters
----------
mail : email
The mail to process.
"""
date = self.getDate(mail)
for part in mail.walk():
ctype = part.get_content_type()
if not ctype in Mail.valid_ctypes: continue
filename = '%s-attachment-%s' % (date, part.get_filename())
filename = self.validateFilename(filename)
filename = os.path.join(self.basepath, filename)
with open(filename, 'wb') as f:
f.write(part.get_payload(decode=True))
def validateFilename(self, filename):
"""
Makes a filename safe.
Parameters
----------
filename : str
Filename to reformat.
Returns
-------
The new filename, stripped of unsafe characters.
"""
return ''.join([c for c in filename if Mail.filesafe.match(c)])
def __del__(self):
"""
Deconstructor disconnects from the IMAP-Server.
"""
self.server.logout()
class ImapDB(BaseDB):
def __init__(self,
username,
password='******',
host='localhost',
port=143,
*args,
**kwargs):
super().__init__(username, *args, **kwargs)
self.imap = IMAPClient(host, port, use_uid=True, ssl=False)
res = self.imap.login(username, password)
self.cursor.execute(
"SELECT lowModSeq,highModSeq,highModSeqMailbox,highModSeqThread,highModSeqEmail FROM account LIMIT 1"
)
row = self.cursor.fetchone()
self.lastfoldersync = 0
if row:
self.lowModSeq,
self.highModSeq,
self.highModSeqMailbox,
self.highModSeqThread,
self.highModSeqEmail = row
else:
self.lowModSeq = 0
self.highModSeq = 1
self.highModSeqMailbox = 1
self.highModSeqThread = 1
self.highModSeqEmail = 1
# (imapname, readonly)
self.selected_folder = (None, False)
self.mailboxes = {}
self.sync_mailboxes()
self.messages = {}
def get_messages_cached(self, properties=(), id__in=()):
messages = []
if not self.messages:
return messages, id__in, properties
fetch_props = set()
fetch_ids = set(id__in)
for id in id__in:
msg = self.messages.get(id, None)
if msg:
found = True
for prop in properties:
try:
msg[prop]
except (KeyError, AttributeError):
found = False
fetch_props.add(prop)
if found:
fetch_ids.remove(id)
messages.append(msg)
# if one messages is missing, need to fetch all properties
if len(messages) < len(id__in):
fetch_props = properties
return messages, fetch_ids, fetch_props
def get_messages(self,
properties=(),
sort={},
inMailbox=None,
inMailboxOtherThan=(),
id__in=None,
threadId__in=None,
**criteria):
# XXX: id == threadId for now
if id__in is None and threadId__in is not None:
id__in = [id[1:] for id in threadId__in]
if id__in is None:
messages = []
else:
# try get everything from cache
messages, id__in, properties = self.get_messages_cached(
properties, id__in=id__in)
fetch_fields = {
f
for prop, f in FIELDS_MAP.items() if prop in properties
}
if 'RFC822' in fetch_fields:
# remove redundand fields
fetch_fields.discard('RFC822.HEADER')
fetch_fields.discard('RFC822.SIZE')
if inMailbox:
mailbox = self.mailboxes.get(inMailbox, None)
if not mailbox:
raise errors.notFound(f'Mailbox {inMailbox} not found')
mailboxes = [mailbox]
elif inMailboxOtherThan:
mailboxes = [
m for m in self.mailboxes.values()
if m['id'] not in inMailboxOtherThan
]
else:
mailboxes = self.mailboxes.values()
search_criteria = as_imap_search(criteria)
sort_criteria = as_imap_sort(sort) or '' if sort else None
mailbox_uids = {}
if id__in is not None:
if len(id__in) == 0:
return messages # no messages matches empty ids
if not fetch_fields and not sort_criteria:
# when we don't need anything new from IMAP, create empty messages
# useful when requested conditions can be calculated from id (threadId)
messages.extend(
self.messages.get(id, 0) or ImapMessage(id=id)
for id in id__in)
return messages
for id in id__in:
# TODO: check uidvalidity
mailboxid, uidvalidity, uid = parse_message_id(id)
uids = mailbox_uids.get(mailboxid, [])
if not uids:
mailbox_uids[mailboxid] = uids
uids.append(uid)
# filter out unnecessary mailboxes
mailboxes = [m for m in mailboxes if m['id'] in mailbox_uids]
for mailbox in mailboxes:
imapname = mailbox['imapname']
if self.selected_folder[0] != imapname:
self.imap.select_folder(imapname, readonly=True)
self.selected_folder = (imapname, True)
uids = mailbox_uids.get(mailbox['id'], None)
# uids are now None or not empty
# fetch all
if sort_criteria:
if uids:
search = f'{",".join(map(str, uids))} {search_criteria}'
else:
search = search_criteria or 'ALL'
uids = self.imap.sort(sort_criteria, search)
elif search_criteria:
if uids:
search = f'{",".join(map(str, uids))} {search_criteria}'
uids = self.imap.search(search)
if uids is None:
uids = '1:*'
fetch_fields.add('UID')
fetches = self.imap.fetch(uids, fetch_fields)
for uid, data in fetches.items():
id = format_message_id(mailbox['id'], mailbox['uidvalidity'],
uid)
msg = self.messages.get(id, None)
if not msg:
msg = ImapMessage(id=id, mailboxIds=[mailbox['id']])
self.messages[id] = msg
for k, v in data.items():
msg[k.decode()] = v
messages.append(msg)
return messages
def changed_record(self, ifolderid, uid, flags=(), labels=()):
res = self.dmaybeupdate(
'imessages', {
'flags': json.dumps(sorted(flags)),
'labels': json.dumps(sorted(labels)),
}, {
'ifolderid': ifolderid,
'uid': uid
})
if res:
msgid = self.dgefield('imessages', {
'ifolderid': ifolderid,
'uid': uid
}, 'msgid')
self.mark_sync(msgid)
def import_message(self, rfc822, mailboxIds, keywords):
folderdata = self.dget('ifolders')
foldermap = {f['ifolderid']: f for f in folderdata}
jmailmap = {
f['jmailboxid']: f
for f in folderdata if f.get('jmailboxid', False)
}
# store to the first named folder - we can use labels on gmail to add to other folders later.
id, others = mailboxIds
imapname = jmailmap[id][imapname]
flags = set(keywords)
for kw in flags:
if kw in KEYWORD2FLAG:
flags.remove(kw)
flags.add(KEYWORD2FLAG[kw])
appendres = self.imap.append('imapname', '(' + ' '.join(flags) + ')',
datetime.now(), rfc822)
# TODO: compare appendres[2] with uidvalidity
uid = appendres[3]
fdata = jmailmap[mailboxIds[0]]
self.do_folder(fdata['ifolderid'], fdata['label'])
ifolderid = fdata['ifolderid']
msgdata = self.dgetone('imessages', {
'ifolderid': ifolderid,
'uid': uid,
}, 'msgid,thrid,size')
# XXX - did we fail to sync this back? Annoying
if not msgdata:
raise Exception(
'Failed to get back stored message from imap server')
# save us having to download it again - drop out of transaction so we don't wait on the parse
message = parse.parse(rfc822, msgdata['msgid'])
self.begin()
self.dinsert(
'jrawmessage', {
'msgid': msgdata['msgid'],
'parsed': json.dumps('message'),
'hasAttachment': message['hasattachment'],
})
self.commit()
return msgdata
def update_messages(self, changes, idmap):
if not changes:
return {}, {}
changed = {}
notchanged = {}
map = {}
msgids = set(changes.keys())
sql = 'SELECT msgid,ifolderid,uid FROM imessages WHERE msgid IN (' + (
('?,' * len(msgids))[:-1]) + ')'
self.cursor.execute(sql, list(msgids))
for msgid, ifolderid, uid in self.cursor:
if not msgid in map:
map[msgid] = {ifolderid: {uid}}
elif not ifolderid in map[msgid]:
map[msgid][ifolderid] = {uid}
else:
map[msgid][ifolderid].add(uid)
msgids.discard(msgid)
for msgid in msgids:
notchanged[msgid] = {
'type': 'notFound',
'description': 'No such message on server',
}
folderdata = self.dget('ifolders')
foldermap = {f['ifolderid']: f for f in folderdata}
jmailmap = {
f['jmailboxid']: f
for f in folderdata if 'jmailboxid' in f
}
jmapdata = self.dget('jmailboxes')
jidmap = {d['jmailboxid']: (d['role'] or '') for d in jmapdata}
jrolemap = {
d['role']: d['jmailboxid']
for d in jmapdata if 'role' in d
}
for msgid in map.keys():
action = changes[msgid]
try:
for ifolderid, uids in map[msgid].items():
# TODO: merge similar actions?
imapname = foldermap[ifolderid]['imapname']
uidvalidity = foldermap[ifolderid]['uidvalidity']
if self.selected_folder != (imapname, False):
self.imap.select_folder(imapname)
self.selected_folder = (imapname, False)
if imapname and uidvalidity and 'keywords' in action:
flags = set(action['keywords'])
for kw in flags:
if kw in KEYWORD2FLAG:
flags.remove(kw)
flags.add(KEYWORD2FLAG[kw])
self.imap.set_flags(uids, flags, silent=True)
if 'mailboxIds' in action:
mboxes = [idmap(k) for k in action['mailboxIds'].keys()]
# existing ifolderids containing this message
# identify a source message to work from
ifolderid = sorted(map[msgid])[0]
uid = sorted(map[msgid][ifolderid])[0]
imapname = foldermap[ifolderid]['imapname']
uidvalidity = foldermap[ifolderid]['uidvalidity']
# existing ifolderids with this message
current = set(map[msgid].keys())
# new ifolderids that should contain this message
new = set(jmailmap[x]['ifolderid'] for x in mboxes)
for ifolderid in new:
# unless there's already a matching message in it
if current.pop(ifolderid):
continue
# copy from the existing message
newfolder = foldermap[ifolderid]['imapname']
self.imap.copy(imapname, uidvalidity, uid, newfolder)
for ifolderid in current:
# these ifolderids didn't exist in new, so delete all matching UIDs from these folders
self.imap.move(
foldermap[ifolderid]['imapname'],
foldermap[ifolderid]['uidvalidity'],
map[msgid][ifolderid], # uids
)
except Exception as e:
notchanged[msgid] = {'type': 'error', 'description': str(e)}
raise e
else:
changed[msgid] = None
return changed, notchanged
def destroy_messages(self, ids):
if not ids:
return [], {}
destroymap = defaultdict(dict)
notdestroyed = {}
idset = set(ids)
rows = self.dget('imessages', {'msgid': ('IN', idset)},
'msgid,ifolderid,uid')
for msgid, ifolderid, uid in rows:
idset.discard(msgid)
destroymap[ifolderid][uid] = msgid
for msgid in idset:
notdestroyed[msgid] = {
'type': 'notFound',
'description': "No such message on server",
}
folderdata = self.dget('ifolders')
foldermap = {d['ifolderid']: d for d in folderdata}
jmailmap = {
d['jmailboxid']: d
for d in folderdata if 'jmailboxid' in d
}
destroyed = []
for ifolderid, ifolder in destroymap.items():
#TODO: merge similar actions?
if not ifolder['imapname']:
for msgid in destroymap[ifolderid]:
notdestroyed[msgid] = \
{'type': 'notFound', 'description': "No folder"}
self.imap.move(ifolder['imapname'], ifolder['uidvalidity'],
destroymap[ifolderid].keys(), None)
destroyed.extend(destroymap[ifolderid].values())
return destroyed, notdestroyed
def deleted_record(self, ifolderid, uid):
msgid = self.dgetfield('imessages', {
'ifolderid': ifolderid,
'uid': uid
}, 'msgid')
if msgid:
self.ddelete('imessages', {'ifolderid': ifolderid, 'uid': uid})
self.mark_sync(msgid)
def get_raw_message(self, msgid, part=None):
self.cursor.execute(
'SELECT imapname,uidvalidity,uid FROM ifolders JOIN imessages USING (ifolderid) WHERE msgid=?',
[msgid])
imapname, uidvalidity, uid = self.cursor.fetchone()
if not imapname:
return None
typ = 'message/rfc822'
if part:
parsed = self.fill_messages([msgid])
typ = find_type(parsed[msgid], part)
res = self.imap.getpart(imapname, uidvalidity, uid, part)
return typ, res['data']
def get_mailboxes(self, fields=None, **criteria):
byimapname = {}
# TODO: LIST "" % RETURN (STATUS (UNSEEN MESSAGES HIGHESTMODSEQ MAILBOXID))
for flags, sep, imapname in self.imap.list_folders():
status = self.imap.folder_status(imapname, ([
'MESSAGES', 'UIDVALIDITY', 'UIDNEXT', 'HIGHESTMODSEQ', 'X-GUID'
]))
flags = [f.lower() for f in flags]
roles = [f for f in flags if f not in KNOWN_SPECIALS]
label = roles[0].decode() if roles else imapname
role = ROLE_MAP.get(label.lower(), None)
can_select = b'\\noselect' not in flags
byimapname[imapname] = {
# Dovecot can fetch X-GUID
'id': status[b'X-GUID'].decode(),
'parentId': None,
'name': imapname,
'role': role,
'sortOrder': 2 if role else (1 if role == 'inbox' else 3),
'isSubscribed': True, # TODO: use LSUB
'totalEmails': status[b'MESSAGES'],
'unreadEmails': 0,
'totalThreads': 0,
'unreadThreads': 0,
'myRights': {
'mayReadItems': can_select,
'mayAddItems': can_select,
'mayRemoveItems': can_select,
'maySetSeen': can_select,
'maySetKeywords': can_select,
'mayCreateChild': True,
'mayRename': False if role else True,
'mayDelete': False if role else True,
'maySubmit': can_select,
},
'imapname': imapname,
'sep': sep.decode(),
'uidvalidity': status[b'UIDVALIDITY'],
'uidnext': status[b'UIDNEXT'],
# Data sync properties
'createdModSeq': status[b'UIDVALIDITY'], # TODO: persist
'updatedModSeq':
status[b'UIDVALIDITY'], # TODO: from persistent storage
'updatedNotCountsModSeq':
status[b'UIDVALIDITY'], # TODO: from persistent storage
'emailHighestModSeq': status[b'HIGHESTMODSEQ'],
'deleted': 0,
}
# set name and parentId for child folders
for imapname, mailbox in byimapname.items():
names = imapname.rsplit(mailbox['sep'], maxsplit=1)
if len(names) == 2:
mailbox['parentId'] = byimapname[names[0]]['id']
mailbox['name'] = names[1]
# update cache
self.mailboxes = {mbox['id']: mbox for mbox in byimapname.values()}
return byimapname.values()
def sync_mailboxes(self):
self.get_mailboxes()
def mailbox_imapname(self, parentId, name):
parent = self.mailboxes.get(parentId, None)
if not parent:
raise errors.notFound('parent folder not found')
return parent['imapname'] + parent['sep'] + name
def create_mailbox(self,
name=None,
parentId=None,
isSubscribed=True,
**kwargs):
if not name:
raise errors.invalidProperties('name is required')
imapname = self.mailbox_imapname(parentId, name)
# TODO: parse returned MAILBOXID
try:
res = self.imap.create_folder(imapname)
except IMAPClientError as e:
desc = str(e)
if '[ALREADYEXISTS]' in desc:
raise errors.invalidArguments(desc)
except Exception:
raise errors.serverFail(res.decode())
if not isSubscribed:
self.imap.unsubscribe_folder(imapname)
status = self.imap.folder_status(imapname, ['UIDVALIDITY'])
self.sync_mailboxes()
return f"f{status[b'UIDVALIDITY']}"
def update_mailbox(self,
id,
name=None,
parentId=None,
isSubscribed=None,
sortOrder=None,
**update):
mailbox = self.mailboxes.get(id, None)
if not mailbox:
raise errors.notFound('mailbox not found')
imapname = mailbox['imapname']
if (name is not None and name != mailbox['name']) or \
(parentId is not None and parentId != mailbox['parentId']):
if not name:
raise errors.invalidProperties('name is required')
newimapname = self.mailbox_imapname(parentId, name)
res = self.imap.rename_folder(imapname, newimapname)
if b'NO' in res or b'BAD' in res:
raise errors.serverFail(res.encode())
if isSubscribed is not None and isSubscribed != mailbox['isSubscribed']:
if isSubscribed:
res = self.imap.subscribe_folder(imapname)
else:
res = self.imap.unsubscribe_folder(imapname)
if b'NO' in res or b'BAD' in res:
raise errors.serverFail(res.encode())
if sortOrder is not None and sortOrder != mailbox['sortOrder']:
# TODO: update in persistent storage
mailbox['sortOrder'] = sortOrder
self.sync_mailboxes()
def destroy_mailbox(self, id):
mailbox = self.mailboxes.get(id, None)
if not mailbox:
raise errors.notFound('mailbox not found')
res = self.imap.delete_folder(mailbox['imapname'])
if b'NO' in res or b'BAD' in res:
raise errors.serverFail(res.encode())
mailbox['deleted'] = datetime.now().timestamp()
self.sync_mailboxes()
def create_submission(self, new, idmap):
if not new:
return {}, {}
todo = {}
createmap = {}
notcreated = {}
for cid, sub in new.items():
msgid = idmap.get(sub['emailId'], sub['emailId'])
if not msgid:
notcreated[cid] = {'error': 'nos msgid provided'}
continue
thrid = self.dgetfield('jmessages', {
'msgid': msgid,
'deleted': 0
}, 'thrid')
if not thrid:
notcreated[cid] = {'error': 'message does not exist'}
continue
id = self.dmake(
'jsubmission', {
'sendat':
datetime.fromisoformat()(sub['sendAt']).isoformat()
if sub['sendAt'] else datetime.now().isoformat(),
'msgid':
msgid,
'thrid':
thrid,
'envelope':
json.dumps(sub['envelope']) if 'envelope' in sub else None,
})
createmap[cid] = {'id': id}
todo[cid] = msgid
self.commit()
for cid, sub in todo.items():
type, rfc822 = self.get_raw_message(todo[cid])
self.imap.send_mail(rfc822, sub['envelope'])
return createmap, notcreated
def update_submission(self, changed, idmap):
return {}, {x: 'change not supported' for x in changed.keys()}
def destroy_submission(self, destroy):
if not destroy:
return [], {}
destroyed = []
notdestroyed = {}
namemap = {}
for subid in destroy:
deleted = self.dgetfield('jsubmission', {'jsubid': subid},
'deleted')
if deleted:
destroy.append(subid)
self.ddelete('jsubmission', {'jsubid': subid})
else:
notdestroyed[subid] = {
'type': 'notFound',
'description': 'submission not found'
}
self.commit()
return destroyed, notdestroyed
def _initdb(self):
super()._initdb()
self.dbh.execute("""
CREATE TABLE IF NOT EXISTS ifolders (
ifolderid INTEGER PRIMARY KEY NOT NULL,
jmailboxid INTEGER,
sep TEXT NOT NULL,
imapname TEXT NOT NULL,
label TEXT,
uidvalidity INTEGER,
uidfirst INTEGER,
uidnext INTEGER,
highestmodseq INTEGER,
uniqueid TEXT,
mtime DATE NOT NULL
)""")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS ifolderj ON ifolders (jmailboxid)")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS ifolderlabel ON ifolders (label)")
self.dbh.execute("""
CREATE TABLE IF NOT EXISTS imessages (
imessageid INTEGER PRIMARY KEY NOT NULL,
ifolderid INTEGER,
uid INTEGER,
internaldate DATE,
modseq INTEGER,
flags TEXT,
labels TEXT,
thrid TEXT,
msgid TEXT,
envelope TEXT,
bodystructure TEXT,
size INTEGER,
mtime DATE NOT NULL
)""")
self.dbh.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS imsgfrom ON imessages (ifolderid, uid)"
)
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS imessageid ON imessages (msgid)")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS imessagethrid ON imessages (thrid)")
self.dbh.execute("""
CREATE TABLE IF NOT EXISTS ithread (
messageid TEXT PRIMARY KEY,
sortsubject TEXT,
thrid TEXT
)""")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS ithrid ON ithread (thrid)")
class IMAPBot(object):
IMAPBotError = IMAPBotError
def __init__(self, host, username, password, ssl=True):
self.server = IMAPClient(host, use_uid=True, ssl=ssl)
self.server.login(username, password)
if 'IDLE' not in self.server.capabilities():
raise IMAPBotError('Sorry, this IMAP server does not support IDLE.')
# the folder where processed emails go
self.processed_folder = 'imapbot_processed'
self.idle_timeout = 5 # seconds
self._is_idle = False
self._run = True
self._create_folder(self.processed_folder)
def check_mail(self):
select_info = self.server.select_folder('INBOX')
print '%d messages in INBOX' % select_info['EXISTS']
messages = self.server.search(['UNSEEN'])
messages = self.server.search(['NOT DELETED'])
print "%d messages that haven't been seen" % len(messages)
if not messages:
return
#response = self.server.fetch(messages, ['FLAGS', 'INTERNALDATE', 'RFC822.SIZE', 'ENVELOPE', 'RFC822.TEXT'])
response = self.server.fetch(messages, ['FLAGS', 'ENVELOPE', 'RFC822.TEXT'])
for message_id, data in response.iteritems():
message = Message(message_id, data['ENVELOPE'], data['RFC822.TEXT'], data['FLAGS'])
self.process(message)
def complete(self, message):
message_ids = [message.id]
self.server.copy(message_ids, self.processed_folder)
self.server.delete_messages(message_ids)
self.server.expunge()
def _create_folder(self, name):
# make sure the folder doesn't already exist
if self.server.folder_exists(name):
return
self.server.create_folder(name)
def handle_message(self, message):
print 'message id: {}, from: {}:'.format(message.id, message.envelope.get_email('from'))
with open('message.txt', 'ab') as fh:
fh.write('{}\n\n'.format(message.text))
print message.plain or message.html or 'no message'
def idle(self):
if self._is_idle:
return
self.server.idle()
self._is_idle = True
return True # this actually changed state
def unidle(self):
if not self._is_idle:
return
self.server.idle_done()
self._is_idle = False
return True # this call actually changed state
def process(self, message):
self.handle_message(message)
self.complete(message)
def run(self):
# process any mail that was in the inbox before coming online
self.check_mail()
# put the connection in idle mode so we get notifications
self.idle()
# loop forever looking for stuff
while self._run:
for message in self.server.idle_check(timeout=self.idle_timeout):
if message[0] == 'OK':
continue
with Unidle(self):
self.check_mail()
def quit(self):
self._run = False
self.unidle()
print self.server.logout()
def main():
# -options-
parser = OptionParser("python %prog [options] CONFIG_FILE")
parser.add_option('-v', '--verbose', action='store_true', dest='verbose', default=False)
(options, args) = parser.parse_args()
# -settings-
if len(args) < 1:
parser.print_help()
exit()
if not os.path.exists(args[0]):
raise IOError(ENOENT, 'Archivo de configuracion no encontrado', args[0])
config = RawConfigParser()
config.read(args[0])
# put read config options (from the .ini) into global namespace and in uppercase
for name, value in config.items('chkemail'):
globals()[name.upper()] = value
# -workflow-
# create, connect, and login to server
server = IMAPClient(HOST, use_uid=True)
try:
server.login(USER, PASSWORD, port=PORT)
except NameError: # if PORT is not defined
server.login(USER, PASSWORD)
inbox = server.select_folder('INBOX')
if options.verbose:
print '%d messages in INBOX (included deleted)' % inbox['EXISTS']
messages = server.search(['NOT DELETED', 'HEADER Content-Type mixed'])
if options.verbose:
print "%d messages with possible attch" % len(messages)
# fetch data from messages, put each message (Mess object) into the msgs list
scan = server.fetch(messages, ['BODYSTRUCTURE', 'ENVELOPE'])
msgs = dict()
for mid, response in scan.iteritems():
# Mess class only works with mulipart messages
if response['BODYSTRUCTURE'].is_multipart:
msgs[mid] = Mess(mid, response['ENVELOPE'], response['BODYSTRUCTURE'])
# Select the messages with attachements I want, put them in group_msgs
group_msgs = dict()
for msg in msgs.itervalues():
group_msgs[msg.id] = list()
for part_num, part_info in msg.parts.iteritems():
if part_info.filename:
filename = part_info.filename.lower()
if filename.endswith('.pdf') or filename.endswith('.xml') or \
filename.endswith('.zip'):
group_msgs[msg.id] += [part_num]
if not group_msgs[msg.id]:
del group_msgs[msg.id]
# fetch all interesting parts
for msg_id, parts in group_msgs.iteritems():
request = ['BODY['+str(part)+']' for part in parts]
response = server.fetch(msg_id, request)
for body_part in response[msg_id].iterkeys():
if 'BODY' in body_part:
msgs[msg_id].parts[body_part[5:-1]].data = response[msg_id][body_part]
# move messages to trash
if len(group_msgs.keys()) > 0:
server.copy(group_msgs.keys(), 'INBOX.Trash')
server.delete_messages(group_msgs.keys())
server.logout()
# ensure there's an OUTPUT_DIR directory
pdf_dir = os.path.join(OUTPUT_PDF, strftime('%Y-%m'))
ensure_dir(pdf_dir)
ensure_dir(OUTPUT_DIR)
# decode and write data to file
num_attch = 0
for msg in msgs.itervalues():
for part in msg.parts:
if part.data:
filename = part.filename.lower()
if filename.endswith('.pdf') or filename.endswith('.xml'):
if filename.endswith('.pdf'):
ensure_dir(os.path.join(pdf_dir, str(msg.envelope)))
attachment_filename = generate_filename(os.path.join(pdf_dir, str(msg.envelope), os.path.basename(part.filename)))
else:
attachment_filename = generate_filename(os.path.join(OUTPUT_DIR, os.path.basename(part.filename)))
with open(attachment_filename, 'wb') as file_:
if part.encoding == 'quoted-printable':
file_.write(decodestring(part.data))
elif part.encoding == 'base64':
file_.write(b64decode(part.data))
num_attch += 1
elif filename.endswith('.zip') and part.encoding == 'base64':
with tempfile.TemporaryFile() as tmp_zip:
tmp_zip.write(b64decode(part.data))
zip_file = ZipFile(tmp_zip)
for f_info in zip_file.infolist():
if f_info.filename.lower().endswith('.xml'):
attachment_filename = generate_filename(os.path.join(OUTPUT_DIR, os.path.basename(f_info.filename)))
elif f_info.filename.lower().endswith('.pdf'):
ensure_dir(os.path.join(pdf_dir, str(msg.envelope)))
attachment_filename = generate_filename(os.path.join(pdf_dir, str(msg.envelope), os.path.basename(f_info.filename)))
else:
continue
with open(attachment_filename, 'wb') as file_:
file_.write(zip_file.read(f_info))
num_attch += 1
if options.verbose:
print '%d files extracted' % num_attch
class IMAP():
"""
Central class for IMAP server communication
"""
Retval = namedtuple('Retval', 'code data')
def __init__(self,
logger,
username,
password,
server='localhost',
port=143,
starttls=False,
imaps=False,
tlsverify=True,
test=False,
timeout=None):
self.logger = logger
self.username = username
self.password = password
self.server = server
self.port = port
self.imaps = imaps
self.starttls = starttls
self.timeout = timeout
self.sslcontext = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
if tlsverify:
self.sslcontext.verify_mode = ssl.CERT_REQUIRED
else:
self.sslcontext.check_hostname = False
self.sslcontext.verify_mode = ssl.CERT_NONE
self.test = test
self.conn = None
def do_select_mailbox(func):
"""
Decorator to do a fresh mailbox SELECT
"""
def wrapper(*args, **kwargs):
if len(args) != 1:
raise AttributeError(
'Size of *args tuple "{0}" isn\'t 1. It looks like you haven\'t specified all '
'method arguments as named arguments!'.format(args))
mailbox = None
for key in ['mailbox', 'source']:
if key in kwargs.keys():
mailbox = kwargs[key]
break
if mailbox is None:
raise KeyError(
'Unable to SELECT a mailbox, kwargs "{0}" doesn\'t contain a mailbox name'
.format(kwargs))
result = args[0].select_mailbox(mailbox)
if not result.code:
raise RuntimeError(result.data)
return func(*args, **kwargs)
return wrapper
def process_error(self, exception, simple_return=False):
"""
Process Python exception by logging a message and optionally showing traceback
"""
trace_info = exc_info()
err_msg = str(exception)
if isinstance(exception, IMAPClient.Error):
err_msg = Helper().byte_to_str(exception.args[0])
self.logger.error("Catching IMAP exception {}: {}".format(
type(exception), err_msg))
if self.logger.isEnabledFor(loglevel_DEBUG):
print_exception(*trace_info)
del trace_info
if simple_return:
return exception
else:
return self.Retval(False, err_msg)
def connect(self, retry=True, logout=False):
"""
Connect to IMAP server and login
"""
if self.starttls:
self.logger.debug(
'Establishing IMAP connection using STARTTLS/{} to {} and logging in with user {}'
.format(self.port, self.server, self.username))
elif self.imaps:
self.logger.debug(
'Establishing IMAP connection using SSL/{} (imaps) to {} and logging in with user {}'
.format(self.port, self.server, self.username))
try:
self.conn = IMAPClient(host=self.server,
port=self.port,
use_uid=True,
ssl=self.imaps,
ssl_context=self.sslcontext,
timeout=self.timeout)
if self.starttls:
self.conn.starttls(ssl_context=self.sslcontext)
login = self.conn.login(self.username, self.password)
login_response = Helper().byte_to_str(login)
# Test login/auth status
login_success = False
noop = self.noop()
if noop.code and noop.data:
login_success = True
if logout:
return self.disconnect()
elif login_success:
return self.Retval(True, login_response)
else:
return self.Retval(False, login_response) # pragma: no cover
except exceptions.LoginError as e:
return self.process_error(e)
except Exception as e:
err_return = self.process_error(e)
if retry:
self.logger.error('Trying one more time to login')
sleep(2)
return self.connect(retry=False, logout=logout)
return err_return
def noop(self):
"""
Do a noop to test login status
"""
try:
noop = self.conn.noop()
noop_response = Helper().byte_to_str(noop[0])
noop_resp_pattern_re = regex_compile('^(Success|NOOP completed)')
login_success = noop_resp_pattern_re.match(noop_response)
return self.Retval(True, login_success)
except IMAPClient.Error as e:
return self.process_error(e)
def disconnect(self):
"""
Disconnect from IMAP server
"""
result = self.conn.logout()
response = Helper().byte_to_str(result)
return self.Retval(response == 'Logging out', response)
def list_mailboxes(self, directory='', pattern='*'):
"""
Get a listing of folders (mailboxes) on the server
"""
try:
raw_list = self.conn.list_folders(directory, pattern)
nice_list = []
for mailbox in raw_list:
flags = []
for flag in mailbox[0]:
flags.append(flag.decode('utf-8'))
nice_list.append({
'name': mailbox[2],
'flags': flags,
'delimiter': mailbox[1].decode("utf-8")
})
return self.Retval(True, nice_list)
except IMAPClient.Error as e:
return self.process_error(e)
def select_mailbox(self, mailbox):
"""
Select a mailbox to work on
"""
self.logger.debug('Switching to mailbox {}'.format(mailbox))
try:
result = self.conn.select_folder(mailbox)
response = {}
for key, value in result.items():
unicode_key = Helper().byte_to_str(key)
if unicode_key == 'FLAGS':
flags = []
for flag in value:
flags.append(Helper().byte_to_str(flag))
response[unicode_key] = tuple(flags)
else:
response[unicode_key] = value
return self.Retval(True, response)
except IMAPClient.Error as e:
return self.process_error(e)
def add_mail(self, mailbox, message, flags=(), msg_time=None):
"""
Add/append a mail to a mailbox
"""
self.logger.debug('Adding a mail into mailbox {}'.format(mailbox))
try:
if not isinstance(message, Mail):
message = Mail(logger=self.logger, mail_native=message)
self.conn.append(mailbox, str(message.get_native()), flags,
msg_time)
# According to rfc4315 we must not return the UID from the response, so we are fetching it ourselves
uids = self.search_mails(mailbox=mailbox,
criteria='HEADER Message-Id "{}"'.format(
message.get_message_id())).data[0]
return self.Retval(True, uids)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def search_mails(self, mailbox, criteria='ALL', autocreate_mailbox=False):
"""
Search for mails in a mailbox
"""
self.logger.debug(
'Searching for mails in mailbox {} and criteria=\'{}\''.format(
mailbox, criteria))
try:
return self.Retval(True, list(self.conn.search(criteria=criteria)))
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def fetch_mails(self, uids, mailbox, return_fields=None):
"""
Retrieve mails from a mailbox
"""
self.logger.debug('Fetching mails with uids {}'.format(uids))
return_raw = True
if return_fields is None:
return_raw = False
return_fields = [b'RFC822']
mails = {}
try:
for uid in uids:
result = self.conn.fetch(uid, return_fields)
if not result:
continue
if return_raw:
mails[uid] = result[uid]
else:
# mails[uid] = Mail(logger=self.logger, uid=uid, mail_native=email.message_from_bytes(result[uid][b'RFC822']))
mails[uid] = Mail(logger=self.logger,
mail_native=email.message_from_bytes(
result[uid][b'RFC822']))
return self.Retval(True, mails)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def get_mailflags(self, uids, mailbox):
"""
Retrieve flags from mails
"""
try:
result = self.conn.get_flags(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to get flags for mail with uid={}: {}'.format(
uid, result))
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def set_mailflags(self, uids, mailbox, flags=[]):
"""
Set and retrieve flags from mails
"""
if self.test:
self.logger.info(
'Would have set mail flags on message uids "{}"'.format(
str(uids)))
return self.Retval(True, None)
else:
self.logger.debug('Setting flags={} on mails uid={}', flags, uids)
try:
result = self.conn.set_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to set and get flags for mail with uid={}: {}'
.format(uid, result))
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def add_mailflags(self, uids, mailbox, flags=[]):
"""
Add and retrieve flags from mails
"""
if self.test:
self.logger.info(
'Would have added mail flags on message uids "{}"'.format(
str(uids)))
return self.Retval(True, None)
else:
self.logger.debug('Adding flags={} on mails uid={}', flags, uids)
try:
result = self.conn.add_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to add and get flags for mail with uid={}: {}'
.format(uid, result))
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def move_mail(self,
message_ids,
source,
destination,
delete_old=True,
expunge=True,
add_flags=None,
set_flags=None):
"""
Move a mail from a mailbox to another
"""
return self.copy_mails(message_ids=message_ids,
source=source,
destination=destination,
delete_old=delete_old,
expunge=expunge,
add_flags=add_flags,
set_flags=set_flags)
@do_select_mailbox
def copy_mails(self,
source,
destination,
message_ids=None,
delete_old=False,
expunge=False,
add_flags=None,
set_flags=None):
"""
Copies one or more mails from a mailbox into another
"""
if self.test:
if delete_old:
self.logger.info(
'Would have moved mail Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode'
.format(message_ids, source, destination))
else:
self.logger.info(
'Would have copied mails with Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode'
.format(message_ids, source, destination))
return self.Retval(True, None)
else:
try:
if delete_old:
self.logger.debug(
'Moving mail Message-Ids="{}" from "{}" to "{}"'.
format(message_ids, source, destination))
else:
self.logger.debug(
'Copying mail Message-Ids="{}" from "{}" to "{}"'.
format(message_ids, source, destination))
# if message_ids is None:
# message_ids = []
# result = self.fetch_mails(uids=uids, mailbox=source)
# if not result.code:
# self.logger.error('Failed to determine Message-Id by uids for mail with uids "{}"', uids)
# return result
# message_ids.append(result.data.keys())
if not self.mailbox_exists(destination).data:
self.logger.info(
'Destination mailbox {} doesn\'t exist, creating it for you'
.format(destination))
result = self.create_mailbox(mailbox=destination)
if not result.code:
self.logger.error(
'Failed to create the mailbox {}: {}'.format(
source, result.data)) # pragma: no cover
return result # pragma: no cover
uids = []
for message_id in message_ids:
result = self.search_mails(
mailbox=source,
criteria='HEADER Message-Id "{}"'.format(message_id))
if not result.code or len(result.data) == 0:
self.logger.error(
'Failed to determine uid by Message-Id for mail with Message-Id "{}"'
.format(message_id))
return self.Retval(False, result.data)
uids.append(result.data[0])
result = self.select_mailbox(source)
if not result.code:
return result # pragma: no cover
self.conn.copy(uids, destination)
if delete_old:
result = self.delete_mails(uids=uids, mailbox=source)
if not result.code:
self.logger.error(
'Failed to remove old mail with Message-Id="{}"/uids="{}": {}'
.format(message_ids, uids,
result.data)) # pragma: no cover
return result # pragma: no cover
if expunge: # TODO don't expunge by default
result = self.expunge(mailbox=source)
if not result.code:
self.logger.error(
'Failed to expunge on mailbox {}: {}'.format(
source, result.data)) # pragma: no cover
return result # pragma: no cover
dest_uids = []
for message_id in message_ids:
result = self.search_mails(
mailbox=destination,
criteria='HEADER Message-Id "{}"'.format(message_id))
if not result.code:
self.logger.error(
'Failed to determine uid by Message-Id for mail with Message-Id "{}"'
.format(message_id)) # pragma: no cover
return result # pragma: no cover
dest_uids.append(result.data[0])
if isinstance(set_flags, list):
self.set_mailflags(uids=dest_uids,
mailbox=destination,
flags=set_flags)
if add_flags:
self.add_mailflags(uids=dest_uids,
mailbox=destination,
flags=add_flags)
return self.Retval(True, dest_uids)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def expunge(self, mailbox):
"""
Expunge mails form a mailbox
"""
self.logger.debug('Expunge mails from mailbox {}'.format(mailbox))
try:
return self.Retval(True, b'Expunge completed.'
in self.conn.expunge())
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
def create_mailbox(self, mailbox):
"""
Create a mailbox
"""
self.logger.debug('Creating mailbox {}'.format(mailbox))
try:
return self.Retval(
True,
self.conn.create_folder(mailbox) == b'Create completed.')
except IMAPClient.Error as e:
return self.process_error(e)
def mailbox_exists(self, mailbox):
"""
Check whether a mailbox exists
"""
try:
return self.Retval(True, self.conn.folder_exists(mailbox))
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
@do_select_mailbox
def delete_mails(self, uids, mailbox):
"""
Delete mails
"""
self.logger.debug('Deleting mails with uid="{}"'.format(uids))
try:
result = self.conn.delete_messages(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to get flags for mail with uid={} after deleting it: {}'
.format(uid, result))
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
class EmailClient(object):
def __init__(self, config):
"""
Create an email client abstraction configured for an account.
config is a dict with keys:
HOST
USERNAME
PASSWORD
USE_SSL default=True
FOLDER default='INBOX'
SELECTORS default=['NOT DELETED']
"""
self.config = {'USE_SLL': True, 'FOLDER': 'INBOX', 'SELECTORS': ['NOT DELETED']}
self.config.update(config)
def connect(self):
self.server = IMAPClient(self.config['HOST'], use_uid=True, ssl=self.config['USE_SSL'])
self.server.login(self.config['USERNAME'], self.config['PASSWORD'])
select_info = self.server.select_folder(self.config['FOLDER'])
def process_new_messages(self, debug=False):
"""
Get mesages from the server.
Note: new information on existing uids will be ignored.
For example, if the rfc_size changes (which is a strangely common occurrence),
the new value will be ignored.
"""
if debug: print "searching"
messages = self.server.search(self.config['SELECTORS'])
if debug: print "done searching"
self._process_messages(messages, debug=debug)
def _process_messages(self, uids, move=False, debug=False):
if debug: print "fetching"
response = self.server.fetch(uids, ['FLAGS', 'RFC822', 'RFC822.SIZE'])
for msg_uid, data in response.iteritems():
if debug: print "processing %s" % msg_uid
with transaction.commit_on_success():
# extract data
flags = data['FLAGS']
rfc_size = data['RFC822.SIZE']
raw_body = data['RFC822']
seq = data['SEQ']
# save objects
email, _ = EmailMessage.objects.get_or_create(
uid = msg_uid,
username = self.config['USERNAME'],
host = self.config['HOST'],
defaults = {
'raw_body': raw_body,
'rfc_size': rfc_size,
'seq': data['SEQ'],
} )
email.save()
for flag in flags:
EmailFlag.objects.get_or_create(email=email, flag=flag)[0].save()
# move message
if move:
self.server.copy([msg_uid], 'auto_processed')
self.server.delete_messages([msg_uid])
class CuckooRequest(object):
def __init__(self, message):
self.message = message
'''cuckooinbox config variables'''
config = Config(cfg=os.path.join(CUCKOO_ROOT,"cuckooinbox","cuckooinbox.conf"))
config = config.get('cuckooinbox')
self.username = config['username']
self.passwd = config['passwd']
self.imap = config['imap']
self.imap_ssl = config['imap_ssl']
self.smtp_server = config['smtp']
self.interval = config['interval']
self.archive_folder = config['archive_folder']
self.email_whitelist = config['email_whitelist']
self.url_limit = config['url_limit']
self.attachment_limit = config['attachment_limit']
self.zip_reports = config['zip_reports']
self.zip_password = config['zip_password']
self.url_blacklist = config['url_blacklist']
self.url_file_backlist = config['url_file_backlist']
self.machine = config['machine']
'''imap variables'''
self.server = IMAPClient(self.imap, use_uid=True, ssl=self.imap_ssl)
self.server.login(self.username, self.passwd)
self.attachment_counter = 0
'''message variables'''
self.msg = MIMEMultipart()
self.response_msg = MIMEMultipart()
self.response_urls = []
self.response_attachments = []
self.sender = ''
self.subject = ''
self.cc_list = []
'''logging object'''
self.log_entry = Logger('cuckooinbox.log')
'''cuckoo variables'''
self.taskids = []
self.db = Database()
self.url_counter = 0 # tracks url count to not exceed url_limit
def fetch(self, message):
'''set retrieve folder'''
select_info = self.server.select_folder('INBOX')
'''fetch mail'''
response = self.server.fetch(self.message, ['RFC822'])
'''parse received email'''
for msgid, data in response.iteritems():
msg_string = data['RFC822']
self.msg = email.message_from_string(msg_string)
'''parse 'Name <*****@*****.**>' format'''
if '<' in self.msg['From']: self.sender = self.msg['From'].split('<'[0])[-1][:-1]
else: self.sender = self.msg['From']
self.subject = self.msg['Subject']
'''print and log successful receive'''
self.log_entry.logEvent('[+] Received email ID: %d from %s [%s]' % (msgid, self.msg['From'], self.msg['Subject']))
'''save CC info for reply later'''
if self.msg['Cc']:
for address in self.msg['Cc'].split(', '): self.cc_list.append(address)
self.log_entry.logEvent( '[*] Email \"%s\" from %s cc\'d the following addresses: %s' % (self.msg['Subject'],self.msg['From'],', '.join(str(copies) for copies in self.cc_list)))
file_whitelist = ['exe', 'doc', 'docx', 'xls', 'xlsx', 'pdf', 'zip']
'''parse message elements'''
for part in self.msg.walk():
if part.get_content_type() == 'text/plain':
self.log_entry.logEvent( '[*] Email ID: %d has a plain text object.' % msgid)
content = part.get_payload()
self.processText(content)
elif part.get_content_type() == 'text/html':
self.log_entry.logEvent('[*] Email ID: %d has a html object.' % msgid)
content = part.get_payload()
self.processText(content)
elif 'application' in part.get_content_type():
# email attachment has no filename
if not part.get_param('name'): return 0
# cuckoo file analysis whitelist
if not part.get_param('name').split('.'[0])[-1] in file_whitelist: break
# increment and break if limit is reached
if (self.attachment_limit != 0 and self.attachment_counter == self.attachment_limit): break
self.attachment_counter += 1
self.log_entry.logEvent('[*] Email ID: %d has an attachment object.' % msgid)
content = part.get_payload()
file_name = part.get_param('name')
self.processAttachment(content, file_name)
'''archive email when done submitting cuckoo tasks'''
try:
self.archive(self.message)
except:
self.server.delete_messages(self.message)
self.server.expunge()
def processText(self, content):
'''reformat quoted string mail to plain html'''
body = quopri.decodestring(content)
soup = BeautifulSoup(body)
# todo analyze href spoof
'''parse and analyze hyperlinks'''
for url in soup.findAll('a'):
# strip mailto links
if url['href'].split(':'[0])[0] == 'mailto' : continue
# strip blacklist links
for item in self.url_blacklist.split(','):
if item in url['href']: return 0
# strip blacklist link filetypes
for item in self.url_file_backlist.split(','):
if item in url['href'].split('.'[0])[-1]: return 0
else:
self.response_urls.append(url['href'])
if self.machine:
task_id = self.db.add_url(url['href'], package="ie", timeout=15, machine=self.machine)
else: task_id = self.db.add_url(url['href'], package="ie", timeout=15)
if task_id:
self.taskids.append(task_id)
self.log_entry.logEvent('[+] URL \"%s\" added as task with ID %d' % (url['href'],task_id))
# increment counter and exit loop if limit is reached
self.url_counter += 1
if (self.url_limit != 0 and self.url_counter == self.url_limit): return 0
else:
self.log_entry.logEvent("[!] Error: adding task to database" % (url['href'],task_id))
break
def processAttachment(self, content, filename):
'''create temp file for analysis'''
temp_file = tempfile.NamedTemporaryFile(prefix=filename.split('.'[0])[0], suffix='.' + filename.split('.'[0])[1])
temp_file.write(content)
self.response_attachments.append(filename)
'''add to cuckoo tasks'''
task_id = self.db.add_path(temp_file.name, timeout=10, package=filename.split('.'[0])[1])
temp_file.flush()
if task_id:
self.taskids.append(task_id)
self.log_entry.logEvent('[+] File \"%s\" added as task with ID %d' % (filename,task_id))
else:
self.taskids.append(task_id)
self.log_entry.logEvent("[!] Error adding task to database")
'''make sure file gets submitted before we toss it'''
timeout = time.time() + 120
while time.time() < timeout:
if os.path.exists(os.path.join(CUCKOO_ROOT,"storage","analyses",str(task_id),"reports","report.html")): continue
time.sleep(.25)
temp_file.close()
def archive(self, message):
select_info = self.server.select_folder('INBOX')
'''cleanup mailbox'''
self.server.copy(self.message,self.archive_folder)
self.server.delete_messages(self.message)
self.server.expunge()
def expunge(self, message):
select_info = self.server.select_folder('INBOX')
'''expunge cuckooinbox request'''
self.server.delete_messages(self.message)
self.server.expunge()
def zipResults(self,):
'''create temporary zip file'''
temp_zip = tempfile.TemporaryFile(prefix='report',suffix='.zip')
zip_file = zipfile.ZipFile(temp_zip, 'w')
if self.zip_password: zip_file.setpassword(self.zip_password)
'''set zip to compress'''
try:
import zlib
compression = zipfile.ZIP_DEFLATED
except:
compression = zipfile.ZIP_STORED
modes = { zipfile.ZIP_DEFLATED: 'deflated',
zipfile.ZIP_STORED: 'stored',}
'''wait for reports to finish then add to list'''
for id in self.taskids:
# timeout error handling
if not os.path.exists(os.path.join(CUCKOO_ROOT,"storage","analyses",str(id),"reports","report.html")):
self.log_entry.logEvent('cuckooinbox error: report timeout reached on task ID %d.' % id)
else:
zip_file.write(os.path.join(CUCKOO_ROOT,"storage","analyses",str(id),"reports","report.html"),\
arcname = 'report' + str(id) + '.html', compress_type=compression)
zip_file.close()
'''attach zip to email message'''
temp_zip.seek(0)
email_file = MIMEBase('application', 'zip')
email_file.set_payload(temp_zip.read())
Encoders.encode_base64(email_file)
email_file.add_header('Content-Disposition', 'attachment; filename="report.zip"')
self.response_msg.attach(email_file)
def sendReport(self,):
'''create email header'''
assert type(self.cc_list)==list
assert type(self.taskids)==list
self.response_msg['From'] = self.username
self.response_msg['To'] = self.sender
self.response_msg['Cc'] = ", ".join(self.cc_list)
self.response_msg['Date'] = formatdate(localtime=True)
self.response_msg['Subject'] = 'cuckooinbox report: ' + self.subject
'''attach cuckooinbox email body'''
for id in self.taskids:
'''wait for reports to finish before sending'''
timeout = time.time() + 120
while time.time() < timeout:
if os.path.exists(os.path.join(CUCKOO_ROOT,"storage","analyses",str(id),"reports","report.html")): continue
time.sleep(.25)
if os.path.exists(os.path.join(CUCKOO_ROOT,"storage","analyses",str(id),"reports","inbox.html")):
file = open(os.path.join(CUCKOO_ROOT,"storage","analyses",str(id),"reports","inbox.html"))
body = '<html>' + \
'<div class="section-title">'+ \
'<h2>Task ID %d <small></small></h2>' % id + \
'</div>'+ \
'<table class="table table-striped table-bordered">'+ \
file.read() + \
'</html>'
file.close()
response_text = ''.join(body)
self.response_msg.attach(MIMEText(response_text,'html'))
else: print '[!] Could not find cuckoobox report files.'
'''wait for analysis to finish and zip the reports'''
self.zipResults()
'''send the message'''
if '@gmail.com' in self.username:
smtp = smtplib.SMTP('smtp.gmail.com',587)
smtp.starttls()
smtp.login(self.username, self.passwd)
else:
smtp = smtplib.SMTP(self.smtp_server)
try: smtp.login(self.username,self.passwd)
except:
self.log_entry.logEvent('[!] SMTP login failed.')
try: smtp.sendmail(self.username, self.sender, self.response_msg.as_string())
except:
self.log_entry.logEvent('SMTP message %s failed to send.' % self.subject)
smtp.close()
self.log_entry.logEvent('[-] Sent "%s" report to %s' % (self.subject, self.sender))
self.server.logout()
def main():
# -options-
parser = OptionParser("python %prog [options] CONFIG_FILE")
parser.add_option('-v',
'--verbose',
action='store_true',
dest='verbose',
default=False)
(options, args) = parser.parse_args()
# -settings-
if len(args) < 1:
parser.print_help()
exit()
if not os.path.exists(args[0]):
raise IOError(ENOENT, 'Archivo de configuracion no encontrado',
args[0])
config = RawConfigParser()
config.read(args[0])
# put read config options (from the .ini) into global namespace and in uppercase
for name, value in config.items('chkemail'):
globals()[name.upper()] = value
# -workflow-
# create, connect, and login to server
server = IMAPClient(HOST, use_uid=True)
try:
server.login(USER, PASSWORD, port=PORT)
except NameError: # if PORT is not defined
server.login(USER, PASSWORD)
inbox = server.select_folder('INBOX')
if options.verbose:
print '%d messages in INBOX (included deleted)' % inbox['EXISTS']
messages = server.search(['NOT DELETED', 'HEADER Content-Type mixed'])
if options.verbose:
print "%d messages with possible attch" % len(messages)
# fetch data from messages, put each message (Mess object) into the msgs list
scan = server.fetch(messages, ['BODYSTRUCTURE', 'ENVELOPE'])
msgs = dict()
for mid, response in scan.iteritems():
# Mess class only works with mulipart messages
if response['BODYSTRUCTURE'].is_multipart:
msgs[mid] = Mess(mid, response['ENVELOPE'],
response['BODYSTRUCTURE'])
# Select the messages with attachements I want, put them in group_msgs
group_msgs = dict()
for msg in msgs.itervalues():
group_msgs[msg.id] = list()
for part_num, part_info in msg.parts.iteritems():
if part_info.filename:
filename = part_info.filename.lower()
if filename.endswith('.pdf') or filename.endswith('.xml') or \
filename.endswith('.zip'):
group_msgs[msg.id] += [part_num]
if not group_msgs[msg.id]:
del group_msgs[msg.id]
# fetch all interesting parts
for msg_id, parts in group_msgs.iteritems():
request = ['BODY[' + str(part) + ']' for part in parts]
response = server.fetch(msg_id, request)
for body_part in response[msg_id].iterkeys():
if 'BODY' in body_part:
msgs[msg_id].parts[
body_part[5:-1]].data = response[msg_id][body_part]
# move messages to trash
if len(group_msgs.keys()) > 0:
server.copy(group_msgs.keys(), 'INBOX.Trash')
server.delete_messages(group_msgs.keys())
server.logout()
# ensure there's an OUTPUT_DIR directory
pdf_dir = os.path.join(OUTPUT_PDF, strftime('%Y-%m'))
ensure_dir(pdf_dir)
ensure_dir(OUTPUT_DIR)
# decode and write data to file
num_attch = 0
for msg in msgs.itervalues():
for part in msg.parts:
if part.data:
filename = part.filename.lower()
if filename.endswith('.pdf') or filename.endswith('.xml'):
if filename.endswith('.pdf'):
ensure_dir(os.path.join(pdf_dir, str(msg.envelope)))
attachment_filename = generate_filename(
os.path.join(pdf_dir, str(msg.envelope),
os.path.basename(part.filename)))
else:
attachment_filename = generate_filename(
os.path.join(OUTPUT_DIR,
os.path.basename(part.filename)))
with open(attachment_filename, 'wb') as file_:
if part.encoding == 'quoted-printable':
file_.write(decodestring(part.data))
elif part.encoding == 'base64':
file_.write(b64decode(part.data))
num_attch += 1
elif filename.endswith('.zip') and part.encoding == 'base64':
with tempfile.TemporaryFile() as tmp_zip:
tmp_zip.write(b64decode(part.data))
zip_file = ZipFile(tmp_zip)
for f_info in zip_file.infolist():
if f_info.filename.lower().endswith('.xml'):
attachment_filename = generate_filename(
os.path.join(
OUTPUT_DIR,
os.path.basename(f_info.filename)))
elif f_info.filename.lower().endswith('.pdf'):
ensure_dir(
os.path.join(pdf_dir, str(msg.envelope)))
attachment_filename = generate_filename(
os.path.join(
pdf_dir, str(msg.envelope),
os.path.basename(f_info.filename)))
else:
continue
with open(attachment_filename, 'wb') as file_:
file_.write(zip_file.read(f_info))
num_attch += 1
if options.verbose:
print '%d files extracted' % num_attch
class gmailPy(object):
def __init__(self):
self.IMAP_SERVER = 'imap.gmail.com'
self.ssl = True
self.myIMAPc = None
self.response = None
self.folders = []
def login(self, username, password):
self.myIMAPc = IMAPClient(self.IMAP_SERVER, ssl=self.ssl)
self.myIMAPc.login(username, password)
# Returns a list of all the folders for a particular account
def get_folders(self):
self.response = self.myIMAPc.list_folders()
for item in self.response:
self.folders.append(item[2].strip('u'))
return self.folders
# Returns the total number of messages in a folder
def get_mail_count(self, folder='Inbox'):
self.response = self.myIMAPc.select_folder(folder, True)
return self.response['EXISTS']
# Method to delete messages based on their size
def delete_bigmail(self, folder='Inbox'):
self.myIMAPc.select_folder(folder, False)
# Gets all the message ids of the messages which are not deleted in the folder
messages = self.myIMAPc.search(['NOT DELETED'])
print "%d messages that aren't deleted" % len(messages)
if len(messages) > 0:
print "You can exit by entering 0 or pressing CTRL+C \n"
else: print "There are no messages in the folder"
# Gets the message sizes for all the message ids returned in previous step
# Note: Just sends one request for all message ids with a return time < 10 ms
self.response = self.myIMAPc.fetch(messages, ['RFC822.SIZE'])
# Sorts the dictionary returned by fetch by size in descending order
sorted_response = sorted(self.response.iteritems(), key=operator.itemgetter(1), reverse=True)
count = 1
try:
for item in sorted_response:
# Gets the biggest message including headers, body, etc.
big_message = self.myIMAPc.fetch(item[0], ['RFC822'])
for msgid, data in big_message.iteritems():
msg_string = data['RFC822']
# Parses the message string using email library
msg = email.message_from_string(msg_string)
val = dict(self.response[msgid])['RFC822.SIZE']
print 'ID %d: From: %s Date: %s' % (msgid, msg['From'], msg['date'])
print 'To: %s' % (msg['To'])
print 'Subject: %s' % (msg['Subject'])
print 'Size: %d bytes \n' % (val)
user_del = raw_input("Do you want to delete this message?(Y/N): ")
if user_del == 'Y':
self.delete_message(msgid)
if count == len(sorted_response):
print "There are no more messages"
else:
print "\nMoving on to the next biggest message >>> \n"
elif user_del == '0':
print "Program exiting"
sys.exit()
else:
if count == len(sorted_response):
print "There are no more messages"
else:
print "\nMoving on to the next biggest message >>> \n"
count += 1
except KeyboardInterrupt:
print "Program exiting"
sys.exit()
# Method to delete messages based on their size with a search criteria
def delete_bigmail_search(self, folder='Inbox', command='', criteria=''):
self.myIMAPc.select_folder(folder, False)
# Gets all the message ids from the server based on the search criteria
messages = self.myIMAPc.search('%s "%s"' % (command, criteria))
print "%d messages that match --> %s: %s" % (len(messages), command, criteria)
if len(messages) > 0:
print "You can exit by entering 0 or pressing CTRL+C \n"
else: print "There are no messages in that matched your search criteria"
# Gets the message sizes for all the message ids returned in previous step
# Note: Just sends one request for all message ids with a return time < 10 ms
self.response = self.myIMAPc.fetch(messages, ['RFC822.SIZE'])
# Sorts the messages in decending order of their sizes
sorted_response = sorted(self.response.iteritems(), key=operator.itemgetter(1), reverse=True)
count = 1
try:
for item in sorted_response:
# Gets the entire content for the biggest message identified
big_message = self.myIMAPc.fetch(item[0], ['RFC822'])
for msgid, data in big_message.iteritems():
msg_string = data['RFC822']
msg = email.message_from_string(msg_string)
val = dict(self.response[msgid])['RFC822.SIZE']
print 'ID %d: From: %s Date: %s' % (msgid, msg['From'], msg['date'])
print 'To: %s' % (msg['To'])
print 'Subject: %s' % (msg['Subject'])
print 'Size: %d bytes \n' % (val)
user_del = raw_input("Do you want to delete this message?(Y/N): ")
if user_del == 'Y':
self.delete_message(msgid)
if count == len(sorted_response):
print "There are no more messages"
else:
print "\nMoving on to the next biggest message >>> \n"
elif user_del == '0':
print "Program exiting"
sys.exit()
else:
if count == len(sorted_response):
print "There are no more messages"
else:
print "\nMoving on to the next biggest message >>> \n"
count += 1
except KeyboardInterrupt:
print "Program exiting"
sys.exit()
# Deletes a message in the current folder based on msg id
def delete_message(self, id):
try:
self.myIMAPc.delete_messages([id])
self.myIMAPc.expunge()
print "Message deleted"
except IMAPClient.Error as err:
print "Message deletion failed"
print err
# Renames a folder
def rename_folder(self, oldfolder, newfolder):
try:
self.myIMAPc.rename_folder(oldfolder, newfolder)
print "Folder %s renamed to %s" % (oldfolder, newfolder)
except IMAPClient.Error as err:
print "Folder renaming failed"
print err
# Creates a new folder
def create_folder(self, folder):
try:
self.myIMAPc.create_folder(folder)
print "New folder %s created" % folder
except IMAPClient.Error as err:
print "Folder creation failed"
print err
# Deletes a folder
def delete_folder(self, folder):
try:
self.myIMAPc.delete_folder(folder)
print "Folder %s deleted" % folder
except IMAPClient.Error as err:
print "Folder deletion failed"
print err
# Creates a new folder and copies the content from the two folders that need to be merged
# Then deletes the old folders
def merge_folders(self, merged_folder, folder_1, folder_2):
try:
self.create_folder(merged_folder)
# Selects the folder with read/write permission
self.myIMAPc.select_folder(folder_1, True)
messages = self.myIMAPc.search(['NOT DELETED'])
print "Moving %d messages from %s to %s" % (len(messages), folder_1, merged_folder)
self.myIMAPc.copy(messages, merged_folder)
self.myIMAPc.select_folder(folder_2, True)
messages = self.myIMAPc.search(['NOT DELETED'])
print "Moving %d messages from %s to %s" % (len(messages), folder_2, merged_folder)
self.myIMAPc.copy(messages, merged_folder)
print "Deleting %s and %s..." % (folder_1, folder_2)
self.delete_folder(folder_1)
self.delete_folder(folder_2)
print "Merge folder operation succeeded"
except IMAPClient.Error as err:
print "Merge operation failed"
print err
def logout(self):
self.myIMAPc.logout()
