def remove_attachments(host, username, password, folder_uids):
server = IMAPClient(host)
server.login(username, password)
# trash_folder = server.find_special_folder(imapclient.TRASH)
for folder_name, uids in folder_uids.items():
server.select_folder(folder_name)
to_fetch = ["RFC822", "FLAGS", "INTERNALDATE"]
fetched = server.fetch(uids, to_fetch)
for uid, msg in fetched.items():
email_message = email.message_from_bytes(msg[b"RFC822"])
final_msg = set_email_payload(email_message)
# server.move(uid, trash_folder)
# server.expunge(uid)
server.delete_messages(uid)
server.append(
folder="INBOX",
msg=final_msg.as_string().encode(),
flags=msg[b"FLAGS"],
msg_time=msg[b"INTERNALDATE"],
)
server.unselect_folder()
server.logout()
def main():
s_username = raw_input("Source Email: ")
s_password = getpass.getpass(prompt="Source Password: "******"Destination Email: ")
d_password = getpass.getpass(prompt="Source Password: "******"Run it for real? (yes/*)")
destination_folder = 'Migrated Chatlogs'
# source server
source = IMAPClient('imap.gmail.com', use_uid=True, ssl=True)
source.login(s_username, s_password)
# destination server
destination = IMAPClient('imap.gmail.com', use_uid=True, ssl=True)
destination.login(d_username, d_password)
select_info = source.select_folder("[Gmail]/Chats", readonly=True)
print 'Migrating %s chat messages' % select_info['EXISTS']
print 'Why don\'t you go grab a cup of coffee.. this is going to take a while'
chats = source.search(['NOT DELETED'])
if not destination.folder_exists(destination_folder):
print "creating %s " % destination_folder
destination.create_folder(destination_folder)
for message in chats:
print message
fetchData = source.fetch(int(message),
['INTERNALDATE', 'FLAGS', 'RFC822'])[message]
if certain == "yes":
destination.append(destination_folder,
msg=fetchData['RFC822'],
flags=fetchData['FLAGS'],
msg_time=fetchData['INTERNALDATE'])
destination.logout()
source.logout()
class DirectImap:
def __init__(self, account: Account) -> None:
self.account = account
self.logid = account.get_config("displayname") or id(account)
self._idling = False
self.connect()
def connect(self):
host = self.account.get_config("configured_mail_server")
port = int(self.account.get_config("configured_mail_port"))
security = int(self.account.get_config("configured_mail_security"))
user = self.account.get_config("addr")
pw = self.account.get_config("mail_pw")
if security == const.DC_SOCKET_PLAIN:
ssl_context = None
else:
ssl_context = ssl.create_default_context()
# don't check if certificate hostname doesn't match target hostname
ssl_context.check_hostname = False
# don't check if the certificate is trusted by a certificate authority
ssl_context.verify_mode = ssl.CERT_NONE
if security == const.DC_SOCKET_STARTTLS:
self.conn = IMAPClient(host, port, ssl=False)
self.conn.starttls(ssl_context)
elif security == const.DC_SOCKET_PLAIN:
self.conn = IMAPClient(host, port, ssl=False)
elif security == const.DC_SOCKET_SSL:
self.conn = IMAPClient(host, port, ssl_context=ssl_context)
self.conn.login(user, pw)
self.select_folder("INBOX")
def shutdown(self):
try:
self.conn.idle_done()
except (OSError, IMAPClientError):
pass
try:
self.conn.logout()
except (OSError, IMAPClientError):
print("Could not logout direct_imap conn")
def create_folder(self, foldername):
try:
self.conn.create_folder(foldername)
except imaplib.IMAP4.error as e:
print("Can't create", foldername, "probably it already exists:", str(e))
def select_folder(self, foldername):
assert not self._idling
return self.conn.select_folder(foldername)
def select_config_folder(self, config_name):
""" Return info about selected folder if it is
configured, otherwise None. """
if "_" not in config_name:
config_name = "configured_{}_folder".format(config_name)
foldername = self.account.get_config(config_name)
if foldername:
return self.select_folder(foldername)
def list_folders(self):
""" return list of all existing folder names"""
assert not self._idling
folders = []
for meta, sep, foldername in self.conn.list_folders():
folders.append(foldername)
return folders
def delete(self, range, expunge=True):
""" delete a range of messages (imap-syntax).
If expunge is true, perform the expunge-operation
to make sure the messages are really gone and not
just flagged as deleted.
"""
self.conn.set_flags(range, [DELETED])
if expunge:
self.conn.expunge()
def get_all_messages(self):
assert not self._idling
# Flush unsolicited responses. IMAPClient has problems
# dealing with them: https://github.com/mjs/imapclient/issues/334
# When this NOOP was introduced, next FETCH returned empty
# result instead of a single message, even though IMAP server
# can only return more untagged responses than required, not
# less.
self.conn.noop()
return self.conn.fetch(ALL, [FLAGS])
def get_unread_messages(self):
assert not self._idling
res = self.conn.fetch(ALL, [FLAGS])
return [uid for uid in res
if SEEN not in res[uid][FLAGS]]
def mark_all_read(self):
messages = self.get_unread_messages()
if messages:
res = self.conn.set_flags(messages, [SEEN])
print("marked seen:", messages, res)
def get_unread_cnt(self):
return len(self.get_unread_messages())
def dump_imap_structures(self, dir, logfile):
assert not self._idling
stream = io.StringIO()
def log(*args, **kwargs):
kwargs["file"] = stream
print(*args, **kwargs)
empty_folders = []
for imapfolder in self.list_folders():
self.select_folder(imapfolder)
messages = list(self.get_all_messages())
if not messages:
empty_folders.append(imapfolder)
continue
log("---------", imapfolder, len(messages), "messages ---------")
# get message content without auto-marking it as seen
# fetching 'RFC822' would mark it as seen.
requested = [b'BODY.PEEK[]', FLAGS]
for uid, data in self.conn.fetch(messages, requested).items():
body_bytes = data[b'BODY[]']
if not body_bytes:
log("Message", uid, "has empty body")
continue
flags = data[FLAGS]
path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
path.mkdir(parents=True, exist_ok=True)
fn = path.joinpath(str(uid))
fn.write_bytes(body_bytes)
log("Message", uid, fn)
email_message = email.message_from_bytes(body_bytes)
log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
if empty_folders:
log("--------- EMPTY FOLDERS:", empty_folders)
print(stream.getvalue(), file=logfile)
def idle_start(self):
""" switch this connection to idle mode. non-blocking. """
assert not self._idling
res = self.conn.idle()
self._idling = True
return res
def idle_check(self, terminate=False):
""" (blocking) wait for next idle message from server. """
assert self._idling
self.account.log("imap-direct: calling idle_check")
res = self.conn.idle_check(timeout=30)
if len(res) == 0:
raise TimeoutError
if terminate:
self.idle_done()
self.account.log("imap-direct: idle_check returned {!r}".format(res))
return res
def idle_wait_for_seen(self):
""" Return first message with SEEN flag
from a running idle-stream REtiurn.
"""
while 1:
for item in self.idle_check():
if item[1] == FETCH:
if item[2][0] == FLAGS:
if SEEN in item[2][1]:
return item[0]
def idle_done(self):
""" send idle-done to server if we are currently in idle mode. """
if self._idling:
res = self.conn.idle_done()
self._idling = False
return res
def append(self, folder, msg):
"""Upload a message to *folder*.
Trailing whitespace or a linebreak at the beginning will be removed automatically.
"""
if msg.startswith("\n"):
msg = msg[1:]
msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
self.conn.append(folder, msg)
def get_uid_by_message_id(self, message_id):
msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
if len(msgs) == 0:
raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
return msgs[0]
class ImapDB(BaseDB):
def __init__(self,
username,
password='******',
host='localhost',
port=143,
*args,
**kwargs):
super().__init__(username, *args, **kwargs)
self.imap = IMAPClient(host, port, use_uid=True, ssl=False)
res = self.imap.login(username, password)
self.cursor.execute(
"SELECT lowModSeq,highModSeq,highModSeqMailbox,highModSeqThread,highModSeqEmail FROM account LIMIT 1"
)
row = self.cursor.fetchone()
self.lastfoldersync = 0
if row:
self.lowModSeq,
self.highModSeq,
self.highModSeqMailbox,
self.highModSeqThread,
self.highModSeqEmail = row
else:
self.lowModSeq = 0
self.highModSeq = 1
self.highModSeqMailbox = 1
self.highModSeqThread = 1
self.highModSeqEmail = 1
# (imapname, readonly)
self.selected_folder = (None, False)
self.mailboxes = {}
self.sync_mailboxes()
self.messages = {}
def get_messages_cached(self, properties=(), id__in=()):
messages = []
if not self.messages:
return messages, id__in, properties
fetch_props = set()
fetch_ids = set(id__in)
for id in id__in:
msg = self.messages.get(id, None)
if msg:
found = True
for prop in properties:
try:
msg[prop]
except (KeyError, AttributeError):
found = False
fetch_props.add(prop)
if found:
fetch_ids.remove(id)
messages.append(msg)
# if one messages is missing, need to fetch all properties
if len(messages) < len(id__in):
fetch_props = properties
return messages, fetch_ids, fetch_props
def get_messages(self,
properties=(),
sort={},
inMailbox=None,
inMailboxOtherThan=(),
id__in=None,
threadId__in=None,
**criteria):
# XXX: id == threadId for now
if id__in is None and threadId__in is not None:
id__in = [id[1:] for id in threadId__in]
if id__in is None:
messages = []
else:
# try get everything from cache
messages, id__in, properties = self.get_messages_cached(
properties, id__in=id__in)
fetch_fields = {
f
for prop, f in FIELDS_MAP.items() if prop in properties
}
if 'RFC822' in fetch_fields:
# remove redundand fields
fetch_fields.discard('RFC822.HEADER')
fetch_fields.discard('RFC822.SIZE')
if inMailbox:
mailbox = self.mailboxes.get(inMailbox, None)
if not mailbox:
raise errors.notFound(f'Mailbox {inMailbox} not found')
mailboxes = [mailbox]
elif inMailboxOtherThan:
mailboxes = [
m for m in self.mailboxes.values()
if m['id'] not in inMailboxOtherThan
]
else:
mailboxes = self.mailboxes.values()
search_criteria = as_imap_search(criteria)
sort_criteria = as_imap_sort(sort) or '' if sort else None
mailbox_uids = {}
if id__in is not None:
if len(id__in) == 0:
return messages # no messages matches empty ids
if not fetch_fields and not sort_criteria:
# when we don't need anything new from IMAP, create empty messages
# useful when requested conditions can be calculated from id (threadId)
messages.extend(
self.messages.get(id, 0) or ImapMessage(id=id)
for id in id__in)
return messages
for id in id__in:
# TODO: check uidvalidity
mailboxid, uidvalidity, uid = parse_message_id(id)
uids = mailbox_uids.get(mailboxid, [])
if not uids:
mailbox_uids[mailboxid] = uids
uids.append(uid)
# filter out unnecessary mailboxes
mailboxes = [m for m in mailboxes if m['id'] in mailbox_uids]
for mailbox in mailboxes:
imapname = mailbox['imapname']
if self.selected_folder[0] != imapname:
self.imap.select_folder(imapname, readonly=True)
self.selected_folder = (imapname, True)
uids = mailbox_uids.get(mailbox['id'], None)
# uids are now None or not empty
# fetch all
if sort_criteria:
if uids:
search = f'{",".join(map(str, uids))} {search_criteria}'
else:
search = search_criteria or 'ALL'
uids = self.imap.sort(sort_criteria, search)
elif search_criteria:
if uids:
search = f'{",".join(map(str, uids))} {search_criteria}'
uids = self.imap.search(search)
if uids is None:
uids = '1:*'
fetch_fields.add('UID')
fetches = self.imap.fetch(uids, fetch_fields)
for uid, data in fetches.items():
id = format_message_id(mailbox['id'], mailbox['uidvalidity'],
uid)
msg = self.messages.get(id, None)
if not msg:
msg = ImapMessage(id=id, mailboxIds=[mailbox['id']])
self.messages[id] = msg
for k, v in data.items():
msg[k.decode()] = v
messages.append(msg)
return messages
def changed_record(self, ifolderid, uid, flags=(), labels=()):
res = self.dmaybeupdate(
'imessages', {
'flags': json.dumps(sorted(flags)),
'labels': json.dumps(sorted(labels)),
}, {
'ifolderid': ifolderid,
'uid': uid
})
if res:
msgid = self.dgefield('imessages', {
'ifolderid': ifolderid,
'uid': uid
}, 'msgid')
self.mark_sync(msgid)
def import_message(self, rfc822, mailboxIds, keywords):
folderdata = self.dget('ifolders')
foldermap = {f['ifolderid']: f for f in folderdata}
jmailmap = {
f['jmailboxid']: f
for f in folderdata if f.get('jmailboxid', False)
}
# store to the first named folder - we can use labels on gmail to add to other folders later.
id, others = mailboxIds
imapname = jmailmap[id][imapname]
flags = set(keywords)
for kw in flags:
if kw in KEYWORD2FLAG:
flags.remove(kw)
flags.add(KEYWORD2FLAG[kw])
appendres = self.imap.append('imapname', '(' + ' '.join(flags) + ')',
datetime.now(), rfc822)
# TODO: compare appendres[2] with uidvalidity
uid = appendres[3]
fdata = jmailmap[mailboxIds[0]]
self.do_folder(fdata['ifolderid'], fdata['label'])
ifolderid = fdata['ifolderid']
msgdata = self.dgetone('imessages', {
'ifolderid': ifolderid,
'uid': uid,
}, 'msgid,thrid,size')
# XXX - did we fail to sync this back? Annoying
if not msgdata:
raise Exception(
'Failed to get back stored message from imap server')
# save us having to download it again - drop out of transaction so we don't wait on the parse
message = parse.parse(rfc822, msgdata['msgid'])
self.begin()
self.dinsert(
'jrawmessage', {
'msgid': msgdata['msgid'],
'parsed': json.dumps('message'),
'hasAttachment': message['hasattachment'],
})
self.commit()
return msgdata
def update_messages(self, changes, idmap):
if not changes:
return {}, {}
changed = {}
notchanged = {}
map = {}
msgids = set(changes.keys())
sql = 'SELECT msgid,ifolderid,uid FROM imessages WHERE msgid IN (' + (
('?,' * len(msgids))[:-1]) + ')'
self.cursor.execute(sql, list(msgids))
for msgid, ifolderid, uid in self.cursor:
if not msgid in map:
map[msgid] = {ifolderid: {uid}}
elif not ifolderid in map[msgid]:
map[msgid][ifolderid] = {uid}
else:
map[msgid][ifolderid].add(uid)
msgids.discard(msgid)
for msgid in msgids:
notchanged[msgid] = {
'type': 'notFound',
'description': 'No such message on server',
}
folderdata = self.dget('ifolders')
foldermap = {f['ifolderid']: f for f in folderdata}
jmailmap = {
f['jmailboxid']: f
for f in folderdata if 'jmailboxid' in f
}
jmapdata = self.dget('jmailboxes')
jidmap = {d['jmailboxid']: (d['role'] or '') for d in jmapdata}
jrolemap = {
d['role']: d['jmailboxid']
for d in jmapdata if 'role' in d
}
for msgid in map.keys():
action = changes[msgid]
try:
for ifolderid, uids in map[msgid].items():
# TODO: merge similar actions?
imapname = foldermap[ifolderid]['imapname']
uidvalidity = foldermap[ifolderid]['uidvalidity']
if self.selected_folder != (imapname, False):
self.imap.select_folder(imapname)
self.selected_folder = (imapname, False)
if imapname and uidvalidity and 'keywords' in action:
flags = set(action['keywords'])
for kw in flags:
if kw in KEYWORD2FLAG:
flags.remove(kw)
flags.add(KEYWORD2FLAG[kw])
self.imap.set_flags(uids, flags, silent=True)
if 'mailboxIds' in action:
mboxes = [idmap(k) for k in action['mailboxIds'].keys()]
# existing ifolderids containing this message
# identify a source message to work from
ifolderid = sorted(map[msgid])[0]
uid = sorted(map[msgid][ifolderid])[0]
imapname = foldermap[ifolderid]['imapname']
uidvalidity = foldermap[ifolderid]['uidvalidity']
# existing ifolderids with this message
current = set(map[msgid].keys())
# new ifolderids that should contain this message
new = set(jmailmap[x]['ifolderid'] for x in mboxes)
for ifolderid in new:
# unless there's already a matching message in it
if current.pop(ifolderid):
continue
# copy from the existing message
newfolder = foldermap[ifolderid]['imapname']
self.imap.copy(imapname, uidvalidity, uid, newfolder)
for ifolderid in current:
# these ifolderids didn't exist in new, so delete all matching UIDs from these folders
self.imap.move(
foldermap[ifolderid]['imapname'],
foldermap[ifolderid]['uidvalidity'],
map[msgid][ifolderid], # uids
)
except Exception as e:
notchanged[msgid] = {'type': 'error', 'description': str(e)}
raise e
else:
changed[msgid] = None
return changed, notchanged
def destroy_messages(self, ids):
if not ids:
return [], {}
destroymap = defaultdict(dict)
notdestroyed = {}
idset = set(ids)
rows = self.dget('imessages', {'msgid': ('IN', idset)},
'msgid,ifolderid,uid')
for msgid, ifolderid, uid in rows:
idset.discard(msgid)
destroymap[ifolderid][uid] = msgid
for msgid in idset:
notdestroyed[msgid] = {
'type': 'notFound',
'description': "No such message on server",
}
folderdata = self.dget('ifolders')
foldermap = {d['ifolderid']: d for d in folderdata}
jmailmap = {
d['jmailboxid']: d
for d in folderdata if 'jmailboxid' in d
}
destroyed = []
for ifolderid, ifolder in destroymap.items():
#TODO: merge similar actions?
if not ifolder['imapname']:
for msgid in destroymap[ifolderid]:
notdestroyed[msgid] = \
{'type': 'notFound', 'description': "No folder"}
self.imap.move(ifolder['imapname'], ifolder['uidvalidity'],
destroymap[ifolderid].keys(), None)
destroyed.extend(destroymap[ifolderid].values())
return destroyed, notdestroyed
def deleted_record(self, ifolderid, uid):
msgid = self.dgetfield('imessages', {
'ifolderid': ifolderid,
'uid': uid
}, 'msgid')
if msgid:
self.ddelete('imessages', {'ifolderid': ifolderid, 'uid': uid})
self.mark_sync(msgid)
def get_raw_message(self, msgid, part=None):
self.cursor.execute(
'SELECT imapname,uidvalidity,uid FROM ifolders JOIN imessages USING (ifolderid) WHERE msgid=?',
[msgid])
imapname, uidvalidity, uid = self.cursor.fetchone()
if not imapname:
return None
typ = 'message/rfc822'
if part:
parsed = self.fill_messages([msgid])
typ = find_type(parsed[msgid], part)
res = self.imap.getpart(imapname, uidvalidity, uid, part)
return typ, res['data']
def get_mailboxes(self, fields=None, **criteria):
byimapname = {}
# TODO: LIST "" % RETURN (STATUS (UNSEEN MESSAGES HIGHESTMODSEQ MAILBOXID))
for flags, sep, imapname in self.imap.list_folders():
status = self.imap.folder_status(imapname, ([
'MESSAGES', 'UIDVALIDITY', 'UIDNEXT', 'HIGHESTMODSEQ', 'X-GUID'
]))
flags = [f.lower() for f in flags]
roles = [f for f in flags if f not in KNOWN_SPECIALS]
label = roles[0].decode() if roles else imapname
role = ROLE_MAP.get(label.lower(), None)
can_select = b'\\noselect' not in flags
byimapname[imapname] = {
# Dovecot can fetch X-GUID
'id': status[b'X-GUID'].decode(),
'parentId': None,
'name': imapname,
'role': role,
'sortOrder': 2 if role else (1 if role == 'inbox' else 3),
'isSubscribed': True, # TODO: use LSUB
'totalEmails': status[b'MESSAGES'],
'unreadEmails': 0,
'totalThreads': 0,
'unreadThreads': 0,
'myRights': {
'mayReadItems': can_select,
'mayAddItems': can_select,
'mayRemoveItems': can_select,
'maySetSeen': can_select,
'maySetKeywords': can_select,
'mayCreateChild': True,
'mayRename': False if role else True,
'mayDelete': False if role else True,
'maySubmit': can_select,
},
'imapname': imapname,
'sep': sep.decode(),
'uidvalidity': status[b'UIDVALIDITY'],
'uidnext': status[b'UIDNEXT'],
# Data sync properties
'createdModSeq': status[b'UIDVALIDITY'], # TODO: persist
'updatedModSeq':
status[b'UIDVALIDITY'], # TODO: from persistent storage
'updatedNotCountsModSeq':
status[b'UIDVALIDITY'], # TODO: from persistent storage
'emailHighestModSeq': status[b'HIGHESTMODSEQ'],
'deleted': 0,
}
# set name and parentId for child folders
for imapname, mailbox in byimapname.items():
names = imapname.rsplit(mailbox['sep'], maxsplit=1)
if len(names) == 2:
mailbox['parentId'] = byimapname[names[0]]['id']
mailbox['name'] = names[1]
# update cache
self.mailboxes = {mbox['id']: mbox for mbox in byimapname.values()}
return byimapname.values()
def sync_mailboxes(self):
self.get_mailboxes()
def mailbox_imapname(self, parentId, name):
parent = self.mailboxes.get(parentId, None)
if not parent:
raise errors.notFound('parent folder not found')
return parent['imapname'] + parent['sep'] + name
def create_mailbox(self,
name=None,
parentId=None,
isSubscribed=True,
**kwargs):
if not name:
raise errors.invalidProperties('name is required')
imapname = self.mailbox_imapname(parentId, name)
# TODO: parse returned MAILBOXID
try:
res = self.imap.create_folder(imapname)
except IMAPClientError as e:
desc = str(e)
if '[ALREADYEXISTS]' in desc:
raise errors.invalidArguments(desc)
except Exception:
raise errors.serverFail(res.decode())
if not isSubscribed:
self.imap.unsubscribe_folder(imapname)
status = self.imap.folder_status(imapname, ['UIDVALIDITY'])
self.sync_mailboxes()
return f"f{status[b'UIDVALIDITY']}"
def update_mailbox(self,
id,
name=None,
parentId=None,
isSubscribed=None,
sortOrder=None,
**update):
mailbox = self.mailboxes.get(id, None)
if not mailbox:
raise errors.notFound('mailbox not found')
imapname = mailbox['imapname']
if (name is not None and name != mailbox['name']) or \
(parentId is not None and parentId != mailbox['parentId']):
if not name:
raise errors.invalidProperties('name is required')
newimapname = self.mailbox_imapname(parentId, name)
res = self.imap.rename_folder(imapname, newimapname)
if b'NO' in res or b'BAD' in res:
raise errors.serverFail(res.encode())
if isSubscribed is not None and isSubscribed != mailbox['isSubscribed']:
if isSubscribed:
res = self.imap.subscribe_folder(imapname)
else:
res = self.imap.unsubscribe_folder(imapname)
if b'NO' in res or b'BAD' in res:
raise errors.serverFail(res.encode())
if sortOrder is not None and sortOrder != mailbox['sortOrder']:
# TODO: update in persistent storage
mailbox['sortOrder'] = sortOrder
self.sync_mailboxes()
def destroy_mailbox(self, id):
mailbox = self.mailboxes.get(id, None)
if not mailbox:
raise errors.notFound('mailbox not found')
res = self.imap.delete_folder(mailbox['imapname'])
if b'NO' in res or b'BAD' in res:
raise errors.serverFail(res.encode())
mailbox['deleted'] = datetime.now().timestamp()
self.sync_mailboxes()
def create_submission(self, new, idmap):
if not new:
return {}, {}
todo = {}
createmap = {}
notcreated = {}
for cid, sub in new.items():
msgid = idmap.get(sub['emailId'], sub['emailId'])
if not msgid:
notcreated[cid] = {'error': 'nos msgid provided'}
continue
thrid = self.dgetfield('jmessages', {
'msgid': msgid,
'deleted': 0
}, 'thrid')
if not thrid:
notcreated[cid] = {'error': 'message does not exist'}
continue
id = self.dmake(
'jsubmission', {
'sendat':
datetime.fromisoformat()(sub['sendAt']).isoformat()
if sub['sendAt'] else datetime.now().isoformat(),
'msgid':
msgid,
'thrid':
thrid,
'envelope':
json.dumps(sub['envelope']) if 'envelope' in sub else None,
})
createmap[cid] = {'id': id}
todo[cid] = msgid
self.commit()
for cid, sub in todo.items():
type, rfc822 = self.get_raw_message(todo[cid])
self.imap.send_mail(rfc822, sub['envelope'])
return createmap, notcreated
def update_submission(self, changed, idmap):
return {}, {x: 'change not supported' for x in changed.keys()}
def destroy_submission(self, destroy):
if not destroy:
return [], {}
destroyed = []
notdestroyed = {}
namemap = {}
for subid in destroy:
deleted = self.dgetfield('jsubmission', {'jsubid': subid},
'deleted')
if deleted:
destroy.append(subid)
self.ddelete('jsubmission', {'jsubid': subid})
else:
notdestroyed[subid] = {
'type': 'notFound',
'description': 'submission not found'
}
self.commit()
return destroyed, notdestroyed
def _initdb(self):
super()._initdb()
self.dbh.execute("""
CREATE TABLE IF NOT EXISTS ifolders (
ifolderid INTEGER PRIMARY KEY NOT NULL,
jmailboxid INTEGER,
sep TEXT NOT NULL,
imapname TEXT NOT NULL,
label TEXT,
uidvalidity INTEGER,
uidfirst INTEGER,
uidnext INTEGER,
highestmodseq INTEGER,
uniqueid TEXT,
mtime DATE NOT NULL
)""")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS ifolderj ON ifolders (jmailboxid)")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS ifolderlabel ON ifolders (label)")
self.dbh.execute("""
CREATE TABLE IF NOT EXISTS imessages (
imessageid INTEGER PRIMARY KEY NOT NULL,
ifolderid INTEGER,
uid INTEGER,
internaldate DATE,
modseq INTEGER,
flags TEXT,
labels TEXT,
thrid TEXT,
msgid TEXT,
envelope TEXT,
bodystructure TEXT,
size INTEGER,
mtime DATE NOT NULL
)""")
self.dbh.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS imsgfrom ON imessages (ifolderid, uid)"
)
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS imessageid ON imessages (msgid)")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS imessagethrid ON imessages (thrid)")
self.dbh.execute("""
CREATE TABLE IF NOT EXISTS ithread (
messageid TEXT PRIMARY KEY,
sortsubject TEXT,
thrid TEXT
)""")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS ithrid ON ithread (thrid)")
class IMAP():
"""
Central class for IMAP server communication
"""
Retval = namedtuple('Retval', 'code data')
def __init__(self,
logger,
username,
password,
server='localhost',
port=143,
starttls=False,
imaps=False,
tlsverify=True,
test=False,
timeout=None):
self.logger = logger
self.username = username
self.password = password
self.server = server
self.port = port
self.imaps = imaps
self.starttls = starttls
self.timeout = timeout
self.sslcontext = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
if tlsverify:
self.sslcontext.verify_mode = ssl.CERT_REQUIRED
else:
self.sslcontext.check_hostname = False
self.sslcontext.verify_mode = ssl.CERT_NONE
self.test = test
self.conn = None
def do_select_mailbox(func):
"""
Decorator to do a fresh mailbox SELECT
"""
def wrapper(*args, **kwargs):
if len(args) != 1:
raise AttributeError(
'Size of *args tuple "{0}" isn\'t 1. It looks like you haven\'t specified all '
'method arguments as named arguments!'.format(args))
mailbox = None
for key in ['mailbox', 'source']:
if key in kwargs.keys():
mailbox = kwargs[key]
break
if mailbox is None:
raise KeyError(
'Unable to SELECT a mailbox, kwargs "{0}" doesn\'t contain a mailbox name'
.format(kwargs))
result = args[0].select_mailbox(mailbox)
if not result.code:
raise RuntimeError(result.data)
return func(*args, **kwargs)
return wrapper
def process_error(self, exception, simple_return=False):
"""
Process Python exception by logging a message and optionally showing traceback
"""
trace_info = exc_info()
err_msg = str(exception)
if isinstance(exception, IMAPClient.Error):
err_msg = Helper().byte_to_str(exception.args[0])
self.logger.error("Catching IMAP exception {}: {}".format(
type(exception), err_msg))
if self.logger.isEnabledFor(loglevel_DEBUG):
print_exception(*trace_info)
del trace_info
if simple_return:
return exception
else:
return self.Retval(False, err_msg)
def connect(self, retry=True, logout=False):
"""
Connect to IMAP server and login
"""
if self.starttls:
self.logger.debug(
'Establishing IMAP connection using STARTTLS/{} to {} and logging in with user {}'
.format(self.port, self.server, self.username))
elif self.imaps:
self.logger.debug(
'Establishing IMAP connection using SSL/{} (imaps) to {} and logging in with user {}'
.format(self.port, self.server, self.username))
try:
self.conn = IMAPClient(host=self.server,
port=self.port,
use_uid=True,
ssl=self.imaps,
ssl_context=self.sslcontext,
timeout=self.timeout)
if self.starttls:
self.conn.starttls(ssl_context=self.sslcontext)
login = self.conn.login(self.username, self.password)
login_response = Helper().byte_to_str(login)
# Test login/auth status
login_success = False
noop = self.noop()
if noop.code and noop.data:
login_success = True
if logout:
return self.disconnect()
elif login_success:
return self.Retval(True, login_response)
else:
return self.Retval(False, login_response) # pragma: no cover
except exceptions.LoginError as e:
return self.process_error(e)
except Exception as e:
err_return = self.process_error(e)
if retry:
self.logger.error('Trying one more time to login')
sleep(2)
return self.connect(retry=False, logout=logout)
return err_return
def noop(self):
"""
Do a noop to test login status
"""
try:
noop = self.conn.noop()
noop_response = Helper().byte_to_str(noop[0])
noop_resp_pattern_re = regex_compile('^(Success|NOOP completed)')
login_success = noop_resp_pattern_re.match(noop_response)
return self.Retval(True, login_success)
except IMAPClient.Error as e:
return self.process_error(e)
def disconnect(self):
"""
Disconnect from IMAP server
"""
result = self.conn.logout()
response = Helper().byte_to_str(result)
return self.Retval(response == 'Logging out', response)
def list_mailboxes(self, directory='', pattern='*'):
"""
Get a listing of folders (mailboxes) on the server
"""
try:
raw_list = self.conn.list_folders(directory, pattern)
nice_list = []
for mailbox in raw_list:
flags = []
for flag in mailbox[0]:
flags.append(flag.decode('utf-8'))
nice_list.append({
'name': mailbox[2],
'flags': flags,
'delimiter': mailbox[1].decode("utf-8")
})
return self.Retval(True, nice_list)
except IMAPClient.Error as e:
return self.process_error(e)
def select_mailbox(self, mailbox):
"""
Select a mailbox to work on
"""
self.logger.debug('Switching to mailbox {}'.format(mailbox))
try:
result = self.conn.select_folder(mailbox)
response = {}
for key, value in result.items():
unicode_key = Helper().byte_to_str(key)
if unicode_key == 'FLAGS':
flags = []
for flag in value:
flags.append(Helper().byte_to_str(flag))
response[unicode_key] = tuple(flags)
else:
response[unicode_key] = value
return self.Retval(True, response)
except IMAPClient.Error as e:
return self.process_error(e)
def add_mail(self, mailbox, message, flags=(), msg_time=None):
"""
Add/append a mail to a mailbox
"""
self.logger.debug('Adding a mail into mailbox {}'.format(mailbox))
try:
if not isinstance(message, Mail):
message = Mail(logger=self.logger, mail_native=message)
self.conn.append(mailbox, str(message.get_native()), flags,
msg_time)
# According to rfc4315 we must not return the UID from the response, so we are fetching it ourselves
uids = self.search_mails(mailbox=mailbox,
criteria='HEADER Message-Id "{}"'.format(
message.get_message_id())).data[0]
return self.Retval(True, uids)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def search_mails(self, mailbox, criteria='ALL', autocreate_mailbox=False):
"""
Search for mails in a mailbox
"""
self.logger.debug(
'Searching for mails in mailbox {} and criteria=\'{}\''.format(
mailbox, criteria))
try:
return self.Retval(True, list(self.conn.search(criteria=criteria)))
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def fetch_mails(self, uids, mailbox, return_fields=None):
"""
Retrieve mails from a mailbox
"""
self.logger.debug('Fetching mails with uids {}'.format(uids))
return_raw = True
if return_fields is None:
return_raw = False
return_fields = [b'RFC822']
mails = {}
try:
for uid in uids:
result = self.conn.fetch(uid, return_fields)
if not result:
continue
if return_raw:
mails[uid] = result[uid]
else:
# mails[uid] = Mail(logger=self.logger, uid=uid, mail_native=email.message_from_bytes(result[uid][b'RFC822']))
mails[uid] = Mail(logger=self.logger,
mail_native=email.message_from_bytes(
result[uid][b'RFC822']))
return self.Retval(True, mails)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def get_mailflags(self, uids, mailbox):
"""
Retrieve flags from mails
"""
try:
result = self.conn.get_flags(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to get flags for mail with uid={}: {}'.format(
uid, result))
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def set_mailflags(self, uids, mailbox, flags=[]):
"""
Set and retrieve flags from mails
"""
if self.test:
self.logger.info(
'Would have set mail flags on message uids "{}"'.format(
str(uids)))
return self.Retval(True, None)
else:
self.logger.debug('Setting flags={} on mails uid={}', flags, uids)
try:
result = self.conn.set_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to set and get flags for mail with uid={}: {}'
.format(uid, result))
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def add_mailflags(self, uids, mailbox, flags=[]):
"""
Add and retrieve flags from mails
"""
if self.test:
self.logger.info(
'Would have added mail flags on message uids "{}"'.format(
str(uids)))
return self.Retval(True, None)
else:
self.logger.debug('Adding flags={} on mails uid={}', flags, uids)
try:
result = self.conn.add_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to add and get flags for mail with uid={}: {}'
.format(uid, result))
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def move_mail(self,
message_ids,
source,
destination,
delete_old=True,
expunge=True,
add_flags=None,
set_flags=None):
"""
Move a mail from a mailbox to another
"""
return self.copy_mails(message_ids=message_ids,
source=source,
destination=destination,
delete_old=delete_old,
expunge=expunge,
add_flags=add_flags,
set_flags=set_flags)
@do_select_mailbox
def copy_mails(self,
source,
destination,
message_ids=None,
delete_old=False,
expunge=False,
add_flags=None,
set_flags=None):
"""
Copies one or more mails from a mailbox into another
"""
if self.test:
if delete_old:
self.logger.info(
'Would have moved mail Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode'
.format(message_ids, source, destination))
else:
self.logger.info(
'Would have copied mails with Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode'
.format(message_ids, source, destination))
return self.Retval(True, None)
else:
try:
if delete_old:
self.logger.debug(
'Moving mail Message-Ids="{}" from "{}" to "{}"'.
format(message_ids, source, destination))
else:
self.logger.debug(
'Copying mail Message-Ids="{}" from "{}" to "{}"'.
format(message_ids, source, destination))
# if message_ids is None:
# message_ids = []
# result = self.fetch_mails(uids=uids, mailbox=source)
# if not result.code:
# self.logger.error('Failed to determine Message-Id by uids for mail with uids "{}"', uids)
# return result
# message_ids.append(result.data.keys())
if not self.mailbox_exists(destination).data:
self.logger.info(
'Destination mailbox {} doesn\'t exist, creating it for you'
.format(destination))
result = self.create_mailbox(mailbox=destination)
if not result.code:
self.logger.error(
'Failed to create the mailbox {}: {}'.format(
source, result.data)) # pragma: no cover
return result # pragma: no cover
uids = []
for message_id in message_ids:
result = self.search_mails(
mailbox=source,
criteria='HEADER Message-Id "{}"'.format(message_id))
if not result.code or len(result.data) == 0:
self.logger.error(
'Failed to determine uid by Message-Id for mail with Message-Id "{}"'
.format(message_id))
return self.Retval(False, result.data)
uids.append(result.data[0])
result = self.select_mailbox(source)
if not result.code:
return result # pragma: no cover
self.conn.copy(uids, destination)
if delete_old:
result = self.delete_mails(uids=uids, mailbox=source)
if not result.code:
self.logger.error(
'Failed to remove old mail with Message-Id="{}"/uids="{}": {}'
.format(message_ids, uids,
result.data)) # pragma: no cover
return result # pragma: no cover
if expunge: # TODO don't expunge by default
result = self.expunge(mailbox=source)
if not result.code:
self.logger.error(
'Failed to expunge on mailbox {}: {}'.format(
source, result.data)) # pragma: no cover
return result # pragma: no cover
dest_uids = []
for message_id in message_ids:
result = self.search_mails(
mailbox=destination,
criteria='HEADER Message-Id "{}"'.format(message_id))
if not result.code:
self.logger.error(
'Failed to determine uid by Message-Id for mail with Message-Id "{}"'
.format(message_id)) # pragma: no cover
return result # pragma: no cover
dest_uids.append(result.data[0])
if isinstance(set_flags, list):
self.set_mailflags(uids=dest_uids,
mailbox=destination,
flags=set_flags)
if add_flags:
self.add_mailflags(uids=dest_uids,
mailbox=destination,
flags=add_flags)
return self.Retval(True, dest_uids)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def expunge(self, mailbox):
"""
Expunge mails form a mailbox
"""
self.logger.debug('Expunge mails from mailbox {}'.format(mailbox))
try:
return self.Retval(True, b'Expunge completed.'
in self.conn.expunge())
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
def create_mailbox(self, mailbox):
"""
Create a mailbox
"""
self.logger.debug('Creating mailbox {}'.format(mailbox))
try:
return self.Retval(
True,
self.conn.create_folder(mailbox) == b'Create completed.')
except IMAPClient.Error as e:
return self.process_error(e)
def mailbox_exists(self, mailbox):
"""
Check whether a mailbox exists
"""
try:
return self.Retval(True, self.conn.folder_exists(mailbox))
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
@do_select_mailbox
def delete_mails(self, uids, mailbox):
"""
Delete mails
"""
self.logger.debug('Deleting mails with uid="{}"'.format(uids))
try:
result = self.conn.delete_messages(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to get flags for mail with uid={} after deleting it: {}'
.format(uid, result))
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
