def create_folder_if_not_exists(client: IMAPClient, folder: str,
logger: logging.Logger):
"""Create folder if it already exists."""
try:
client.create_folder(folder)
except IMAPClientError:
# most likely, it already exists
logger.info("Looks like the folder %s already exists.", folder)
def __init__(self, HOST, USERNAME, PASSWORD, ssl=False,
folder='TODOLIST'):
imap = IMAPClient(HOST, use_uid=True, ssl=ssl)
imap.login(USERNAME, PASSWORD)
try:
imap.select_folder(folder)
except imap.Error, e:
imap.create_folder(folder)
def random_folder(logged_in_client: IMAPClient):
folder = "random-122122121"
if not logged_in_client.folder_exists(folder):
logged_in_client.create_folder(folder)
yield folder
if logged_in_client.folder_exists(folder):
logged_in_client.delete_folder(folder)
assert not logged_in_client.folder_exists(folder)
def test001_imapclient_can_create_folder_in_imap():
"""
Client can create folders in his mail.
Steps:
- Start imap server, should succeed.
- List default folder, inbox should be there.
- Create new folder, should succeed.
"""
global servers
servers = []
cmd = "kosmos 'j.servers.imap.start()'"
info("Execute {} in tmux main session".format(cmd))
pan = j.servers.tmux.execute(cmd=cmd)
info("Wait for 30s to make sure that the server is running")
sleep(30)
info("Assert that the server is running")
assert pan.cmd_running is True, "imap server is not running"
servers.append(pan)
info("List default folder, inbox should be there")
box = Imbox("0.0.0.0", "*****@*****.**", "randomPW", ssl=False, port=7143)
# assert "INBOX" in str(box.folders()[-1][0])
# assert the whole string instead of the first element in the tuple as it is ordered alphabetically.
assert "INBOX" in str(box.folders()[-1])
info("Connect the client to the IMAP server")
client = IMAPClient("0.0.0.0", port=7143, ssl=False)
client.login("*****@*****.**", "randomPW")
box_name = j.data.idgenerator.generateXCharID(10)
info("Create {} box".format(box_name))
client.create_folder(box_name)
info("Assert that the new box has been created")
# assert box_name in str(box.folders()[-1][0])
# assert the whole string instead of the first element in the tuple as it is ordered alphabetically.
assert box_name in str(box.folders()[-1])
def main():
s_username = raw_input("Source Email: ")
s_password = getpass.getpass(prompt="Source Password: "******"Destination Email: ")
d_password = getpass.getpass(prompt="Source Password: "******"Run it for real? (yes/*)")
destination_folder = 'Migrated Chatlogs'
# source server
source = IMAPClient('imap.gmail.com', use_uid=True, ssl=True)
source.login(s_username, s_password)
# destination server
destination = IMAPClient('imap.gmail.com', use_uid=True, ssl=True)
destination.login(d_username, d_password)
select_info = source.select_folder("[Gmail]/Chats", readonly=True)
print 'Migrating %s chat messages' % select_info['EXISTS']
print 'Why don\'t you go grab a cup of coffee.. this is going to take a while'
chats = source.search(['NOT DELETED'])
if not destination.folder_exists(destination_folder):
print "creating %s " % destination_folder
destination.create_folder(destination_folder)
for message in chats:
print message
fetchData = source.fetch(int(message),
['INTERNALDATE', 'FLAGS', 'RFC822'])[message]
if certain == "yes":
destination.append(destination_folder,
msg=fetchData['RFC822'],
flags=fetchData['FLAGS'],
msg_time=fetchData['INTERNALDATE'])
destination.logout()
source.logout()
class MailGetter(object):
"""
Encapsulates behavior of retrieving email, saving PDFs, and moving emails.
"""
def __init__(self):
"""
Class initializer.
"""
mimetypes.init()
logger = Logger()
self.logger = logger.get_logger(MAIN + ".Cls")
self.server = None
self.queue = BotQueue()
if not _check_paths(['input_path']):
self.logger.fatal("File paths could not be created...cannot continue.")
exit()
def connect(self) -> bool:
"""
Connect and login to the remote IMAP server.
Returns:
(bool): True if successful, otherwise False
"""
mailserver = os.environ.get('mailserver')
imapport = os.environ.get('imapport')
mailssl = os.environ.get('mailssl')
try:
self.server = IMAPClient(
mailserver,
port=imapport,
ssl=mailssl,
use_uid=True
)
username = os.environ.get('mail_username')
password = os.environ.get('mail_password')
self.logger.debug(f"Username: {username}, Password: {password}")
self.server.login(username, password)
except ConnectionRefusedError as e:
self.logger.fatal(f"Connection to {mailserver}:{imapport} was refused: {str(e)}")
return False
except Exception as e:
self.logger.fatal(f"Connection to {mailserver}:{imapport} was refused: {str(e)}")
return False
return True
def reconnect(self) -> bool:
"""
Reconnect to the email server.
Returns:
(bool): True if successful, otherwise False
"""
self.logger.debug("Reconnecting to server.")
try:
self.server.idle_done()
except Exception as e:
self.logger.error("Error exiting IDLE mode: %s", e)
self.logger.info("Will try to reconnect anyway.")
try:
self.server.logout()
except Exception as e:
self.logger.error("Error disconnecting from IMAP server: %s", e)
self.logger.info("Will try to reconnect anyway.")
return self.connect()
def check_folders(self) -> bool:
"""
Check to see if we have the required INBOX and PROCESSED folders.
If we do not have an inbox, we can't go on.
If we do not have a processed folder, try to create it.
Returns:
(bool): True if successful, otherwise false.
"""
# First, make sure the INBOX folder exists. If it does not exist,
# we have a serious problem and need to quit.
inbox = os.environ.get('inbox')
if not self.server.folder_exists(inbox):
self.logger.fatal(f"Error locating INBOX named '{inbox}'.")
return False
# Next, see if the PROCESSED folder exists. If it does not, try to
# create it. If we try to create it and the creation fails, again,
# we have a serious problem and cannot continue.
processed_folder = os.environ.get('processed_folder')
if not self.server.folder_exists(processed_folder):
self.logger.error(f"Error locating PROCESSED folder named '{processed_folder}'.")
self.logger.error(f"Will attempt to create folder named '{processed_folder}'.")
try:
message = self.server.create_folder(processed_folder)
self.logger.info(
"Successfully created '%s': %s",
processed_folder,
message
)
except Exception:
self.logger.fatal(
"Failed to create '%s': %s",
processed_folder,
message
)
return False
self.logger.info("Folder check was successful.")
return True
def save_linked_files(
self,
links: list,
msgid: str,
from_email: str,
subject: str,
reply_to: str
):
"""
Download and save files referenced by a link instead of directly
attached to the email message. Raises exceptions rather than catches
them so that the caller's error handler can deal with and record
the error. Setting *allow_redirects* to ```True``` let's us retrieve
files from cloud services such as DropBox and from URL smashers like
www.tinyurl.com.
Args:
links (list): List of links to download from.
msgid (str): ID of the email message we are processing. Used for
filename disambiguation.
from_email (str): Apparent sender of the email.
subject (str): Subject line of the email.
reply_to (str): Reply-To Address of the email
"""
input_path = os.environ.get('input_path')
for link in links:
if link[-4:].upper() == ".PDF":
my_link = cloudize_link(link)
self.logger.debug("Found link: %s", my_link)
content = requests.get(my_link, allow_redirects=True).content
filename = "{}/{}-{}".format(
input_path,
msgid,
urllib.parse.unquote(link[link.rfind("/")+1:])
)
with open(filename, "wb") as fp:
fp.write(content)
self.queue.publish(arrival_notification(
from_email, reply_to, subject, filename, 'application/pdf')
)
def process_message(self, msgid, message) -> bool:
"""
Process one message.
Args:
msgid (str): Unique ID for this message.
message (email): Email message to process.
Returns:
(bool): True if successful, otherwise False
"""
self.logger.debug(
"ID #%s: From: %s; Subject: %s",
msgid,
message.get("From"),
message.get("Subject")
)
from_email = sanitize_from_name(message.get("From"))
reply_to = sanitize_from_name(message.get("Return-Path") or from_email)
input_path = os.environ.get('input_path')
try:
subject = message.get("Subject")
except OSError as e:
self.logger.error("Cannot extract message subject: %s", str(e))
subject = "N/A"
for key in message.keys():
self.logger.debug("%s = %s", key, message.get(key, None))
# Go through each part of the message. If we find a pdf file, save it.
counter = 1 # number of attachments we've processed for this message.
for part in message.walk():
try:
# multipart/* are just containers...skip them
if part.get_content_maintype() == "multipart":
continue
# Extract & sanitize filename. Create a filename if not given.
filename = part.get_filename()
self.logger.debug("*** %s ***", filename)
extension = mimetypes.guess_extension(part.get_content_type())\
or ".bin"
if not filename:
filename = "{}-{}-part-{}{}".format(
msgid,
from_email,
counter,
extension
)
else:
filename = "{}-{}"\
.format(msgid, filename)\
.replace("\r", "")\
.replace("\n", "")
counter += 1
# Save the attached file
# For now, only save files of type ".PDF"
# TODO: Parse HTML parts to see if we have links to PDF files
# stored elsewhere.
lower_extension = extension.lower()
# Save attached file . . .
if lower_extension in ['.pdf', '.docx', '.doc', '.rtf']:
try:
mimetype = mimetypes.types_map[lower_extension]
except KeyError as error:
self.logger.error(
"Cannot map '%s' to a mime type: %s",
lower_extension,
str(error)
)
mimetype = None
if mimetype is not None:
filename = "{}/{}".format(
input_path,
filename
)
with open(filename, "wb") as fp:
fp.write(part.get_payload(decode=True))
self.queue.publish(arrival_notification(
from_email,
reply_to,
subject,
filename,
mimetype)
)
# Save file referenced by a link . . .
elif lower_extension[:4] == ".htm":
links = extract_html_links(part.get_payload())
self.save_linked_files(
links,
msgid,
from_email,
subject,
reply_to
)
elif lower_extension == ".bat":
links = extract_text_links(part.get_payload())
self.save_linked_files(
links,
msgid,
from_email,
subject,
reply_to
)
else:
self.logger.info("Skipping: %s", filename)
except Exception as e:
self.logger.error(
"Error with attachment #%s from message #%s from %s: %s",
counter, msgid, message.get("From"), e
)
self.logger.exception(e)
return False
return True
def wait_for_messages(self):
# FIRST: Process anything that's already in our INBOX
self.process_inbox()
# NEXT: Go into IDLE mode waiting for either a timeout or new mail.
self.server.idle()
stay_alive = True
# Number of times we've returned from idle without receiving any
# messages.
idle_counter = 0
# Number of seconds to wait for more messages before timing out.
idle_timeout = 60
# If we don't receive something within this many seconds, we'll
# reconnect to the server.
reconnect_seconds = 300
while stay_alive:
try:
# Wait for up to *idle_timeout* seconds for new messages.
responses = self.server.idle_check(timeout=idle_timeout)
self.logger.debug("Response to idle_check(): %s", responses)
if responses:
# We DID get new messages. Process them.
idle_counter = 0
responses = self.server.idle_done()
self.logger.debug("Response to idle_done(): %s", responses)
self.process_inbox()
self.server.idle()
else:
# We did not get any new messages.
idle_counter += 1
# If we've run out of patience, reconnect and resume idle
# mode.
if idle_counter * idle_timeout > reconnect_seconds:
if self.reconnect():
idle_counter = 0
self.process_inbox()
self.server.idle()
# Reconnect failure (!)
else:
stay_alive = False
except imaplib.IMAP4.abort as e:
self.logger.error("IMAP connection closed by host: %s", e)
self.logger.error("Will try to reconnect")
if self.reconnect():
idle_counter = 0
self.process_inbox()
self.server.idle()
else:
self.logger.error("Unable to reconnect. Shutting down.")
stay_alive = False
except Exception as e:
self.logger.error("Error in IDLE loop: %s", e)
self.logger.exception(e)
self.logger.error("Shutting down due to the above errors.")
stay_alive = False
def process_inbox(self):
"""
Process each message in the INBOX. After a message is processed:
If processing was successful: Mark as "SEEN" so that we don't
process it again.
If processing was not successful: Mark as "SEEN" and "FOLLOWUP" so
that an operator can fix if and requeue it by clearing the SEEN
flag.
"""
select_info = self.server.select_folder(os.environ.get('inbox'))
self.logger.debug(
"%d messages in %s.",
select_info[b'EXISTS'],
os.environ.get('inbox')
)
messages = self.server.search(criteria='UNSEEN')
for msgid, data in self.server.fetch(messages, ['RFC822']).items():
email_message = email.message_from_bytes(data[b'RFC822'])
if self.process_message(msgid, email_message):
# Mark message as "Seen". For now, we *won't* move the message
# to the PROCESSED folder.
self.server.set_flags(msgid, b'\\Seen')
else:
# If we had an error processing the message, mark it for
# follow AND as seen.
self.server.set_flags(
msgid,
[b'\\Flagged for Followup', b'\\Seen']
)
class SourceDriverMail(SourceDriver):
def __init__(self, instanceName: str, settings: Settings,
parser: MessageParser) -> None:
super().__init__("mail", instanceName, settings, parser)
# Settings
self.__server = self.getSettingString("server", "")
self.__user = self.getSettingString("user", "")
self.__password = self.getSettingString("password", "")
self.__ssl = self.getSettingBoolean("ssl", True)
self.__fix_weak_dh = self.getSettingBoolean("fix_weak_dh", False)
self.__allowlist = self.getSettingList("allowlist", [])
self.__denylist = self.getSettingList("denylist", [])
self.__cleanup = self.getSettingString("cleanup", "")
self.__archive_folder = self.getSettingString("archive_folder",
"Archive")
if self.__cleanup == "Delete":
self.print("Cleanup strategy is delete")
elif self.__cleanup == "Archive":
self.print("Cleanup strategy is archive")
elif self.__cleanup == "":
self.print("Cleanup is disabled")
else:
self.fatal("Unknown cleanup strategy")
# Internal
self.__healthy = False
self.__connect()
def retrieveEvent(self) -> Optional[SourceEvent]:
try:
if self.isDebug():
self.print("Checking for new mails")
messages = self.__imap_client.search('UNSEEN')
for uid, message_data in self.__imap_client.fetch(
messages, "RFC822").items():
message = email.message_from_bytes(message_data[b"RFC822"],
policy=policy.default)
sender = parseaddr(message.get("From"))[1]
sourceEvent = SourceEvent()
sourceEvent.source = SourceEvent.SOURCE_MAIL
sourceEvent.timestamp = datetime.datetime.strptime(
message.get('Date'), "%a, %d %b %Y %H:%M:%S %z").strftime(
SourceEvent.TIMESTAMP_FORMAT)
sourceEvent.sender = sender
sourceEvent.raw = message.get_body(('plain', )).get_content()
if self.isSenderAllowed(allowlist=self.__allowlist,
denylist=self.__denylist,
sender=sender):
parsedSourceEvent = self.parser.parseMessage(
sourceEvent, None) # type: ignore[union-attr]
self.__do_cleanup(uid)
return parsedSourceEvent
else:
self.error("Received unhandled message (ignored sender)")
return UnhandledEvent.fromSourceEvent(
sourceEvent, UnhandledEvent.CAUSE_IGNORED_SENDER)
except (timeout, OSError) as e:
self.error("Connection to mailserver timed out")
self.__healthy = False
self.__connect()
def getSourceState(self) -> SourceState:
if self.__healthy:
return SourceState.OK
return SourceState.ERROR
def __connect(self):
if self.__fix_weak_dh:
context = ssl.SSLContext(
ssl.PROTOCOL_TLSv1_2) # Workaround for weak dh key
context.set_ciphers('DEFAULT@SECLEVEL=1')
else:
context = ssl.SSLContext()
try:
if self.isDebug():
self.print("Connecting to server {}".format(self.__server))
self.__imap_client = IMAPClient(self.__server,
use_uid=True,
ssl=self.__ssl,
ssl_context=context,
timeout=1.0)
except gaierror:
self.error("Failed to connect to Mail Server")
except ssl.SSLError:
self.fatal("Failed to connect to Mail Server (TLS Error)")
else:
try:
if self.isDebug():
self.print("Login as user {}".format(self.__user))
self.__imap_client.login(self.__user, self.__password)
except LoginError:
self.error("Mail Server login failed")
else:
if self.isDebug():
self.print("Login successful")
self.__healthy = True
self.__imap_client.select_folder('INBOX', readonly=False)
def __create_imap_folder(self, folder):
if not self.__imap_client.folder_exists(folder):
self.print("Folder {} does not exist creating")
self.__imap_client.create_folder(folder)
def __do_cleanup(self, uid):
if self.__cleanup == "Archive":
self.__create_imap_folder(self.__archive_folder)
self.__imap_client.copy(uid, self.__archive_folder)
if self.__cleanup == "Delete" or self.__cleanup == "Archive":
self.__imap_client.delete_messages(uid)
self.__imap_client.expunge(uid)
class ImapConn(object):
def __init__(self, db, foldername, conn_info):
self.db = db
self.foldername = foldername
self._thread = None
self.MHOST, self.MUSER, self.MPASSWORD = conn_info
self.event_initial_polling_complete = threading.Event()
self.pending_imap_jobs = False
# persistent database state below
self.db_folder = self.db.setdefault(foldername, {})
self.db_messages = self.db.setdefault(":message-full", {})
last_sync_uid = db_folder_attr("last_sync_uid")
@contextlib.contextmanager
def wlog(self, msg):
t = time.time() - started
with lock_log:
print("%03.2f [%s] %s -->" % (t, self.foldername, msg))
t0 = time.time()
yield
t1 = time.time()
with lock_log:
print("%03.2f [%s] ... finish %s (%3.2f secs)" %
(t1 - started, self.foldername, msg, t1 - t0))
def log(self, *msgs):
t = time.time() - started
bmsg = "%03.2f [%s]" % (t, self.foldername)
with lock_log:
print(bmsg, *msgs)
def connect(self):
with self.wlog("IMAP_CONNECT {}: {}".format(self.MUSER,
self.MPASSWORD)):
ssl_context = ssl.create_default_context()
# don't check if certificate hostname doesn't match target hostname
ssl_context.check_hostname = False
# don't check if the certificate is trusted by a certificate authority
ssl_context.verify_mode = ssl.CERT_NONE
self.conn = IMAPClient(self.MHOST, ssl_context=ssl_context)
self.conn.login(self.MUSER, self.MPASSWORD)
self.log(self.conn.welcome)
try:
self.select_info = self.conn.select_folder(self.foldername)
except IMAPClientError:
self.ensure_folder_exists()
self.select_info = self.conn.select_folder(self.foldername)
self.log('folder has %d messages' % self.select_info[b'EXISTS'])
self.log('capabilities', self.conn.capabilities())
def ensure_folder_exists(self):
with self.wlog("ensure_folder_exists: {}".format(self.foldername)):
try:
resp = self.conn.create_folder(self.foldername)
except IMAPClientError as e:
if "ALREADYEXISTS" in str(e):
return
print("EXCEPTION:" + str(e))
else:
print("Server sent:", resp if resp else "nothing")
def move(self, messages):
self.log("IMAP_MOVE to {}: {}".format(MVBOX, messages))
try:
resp = self.conn.move(messages, MVBOX)
except IMAPClientError as e:
if "EXPUNGEISSUED" in str(e):
self.log(
"IMAP_MOVE errored with EXPUNGEISSUED, probably another client moved it"
)
else:
self.log(
"IMAP_MOVE {} successfully completed.".format(messages))
def perform_imap_idle(self):
if self.pending_imap_jobs:
self.log("perform_imap_idle skipped because jobs are pending")
return
with self.wlog("IMAP_IDLE()"):
res = self.conn.idle()
interrupted = False
while not interrupted:
# Wait for up to 30 seconds for an IDLE response
responses = self.conn.idle_check(timeout=30)
self.log("Server sent:", responses if responses else "nothing")
for resp in responses:
if resp[1] == b"EXISTS":
# we ignore what is returned and just let
# perform_imap_fetch look since lastseen
# id = resp[0]
interrupted = True
resp = self.conn.idle_done()
def perform_imap_fetch(self):
range = "%s:*" % (self.last_sync_uid + 1, )
with self.wlog("IMAP_PERFORM_FETCH %s" % (range, )):
requested_fields = [
b"RFC822.SIZE", b'FLAGS',
b"BODY.PEEK[HEADER.FIELDS (FROM TO CC DATE CHAT-VERSION MESSAGE-ID IN-REPLY-TO)]"
]
resp = self.conn.fetch(range, requested_fields)
timestamp_fetch = time.time()
for uid in sorted(resp): # get lower uids first
if uid < self.last_sync_uid:
self.log(
"IMAP-ODDITY: ignoring bogus uid %s, it is lower than min-requested %s"
% (uid, self.last_sync_uid))
continue
data = resp[uid]
headers = data[requested_fields[-1].replace(b'.PEEK', b'')]
msg_headers = email.message_from_bytes(headers)
message_id = normalized_messageid(msg_headers)
chat_version = msg_headers.get("Chat-Version")
in_reply_to = msg_headers.get("In-Reply-To", "").lower()
if not self.has_message(normalized_messageid(msg_headers)):
self.log('fetching body of ID %d: %d bytes, message-id=%s '
'in-reply-to=%s chat-version=%s' % (
uid,
data[b'RFC822.SIZE'],
message_id,
in_reply_to,
chat_version,
))
fetchbody_resp = self.conn.fetch(uid, [b'BODY.PEEK[]'])
msg = email.message_from_bytes(
fetchbody_resp[uid][b'BODY[]'])
msg.fetch_retrieve_time = timestamp_fetch
msg.foldername = self.foldername
msg.uid = uid
msg.move_state = DC_CONSTANT_MSG_MOVESTATE_PENDING
self.store_message(message_id, msg)
else:
msg = self.get_message_from_db(message_id)
self.log('fetching-from-db: ID %s message-id=%s' %
(uid, message_id))
if msg.foldername != self.foldername:
self.log("detected moved message", message_id)
msg.foldername = self.foldername
msg.move_state = DC_CONSTANT_MSG_MOVESTATE_STAY
if self.foldername in (INBOX, SENT):
if self.resolve_move_status(
msg) != DC_CONSTANT_MSG_MOVESTATE_PENDING:
# see if there are pending messages which have a in-reply-to
# to our currnet msg
# NOTE: should be one sql-statement to find the
# possibly multiple messages that waited on us
for dbmid, dbmsg in self.db_messages.items():
if dbmsg.move_state == DC_CONSTANT_MSG_MOVESTATE_PENDING:
if dbmsg.get("In-Reply-To",
"").lower() == message_id:
self.log("resolving pending message",
dbmid)
# resolving the dependent message must work now
res = self.resolve_move_status(dbmsg)
assert res != DC_CONSTANT_MSG_MOVESTATE_PENDING, (
dbmid, res)
if not self.has_message(message_id):
self.store_message(message_id, msg)
self.last_sync_uid = max(uid, self.last_sync_uid)
self.log("last-sync-uid after fetch:", self.last_sync_uid)
self.db.sync()
def resolve_move_status(self, msg):
""" Return move-state after this message's next move-state is determined (i.e. it is not PENDING)"""
message_id = normalized_messageid(msg)
if msg.move_state == DC_CONSTANT_MSG_MOVESTATE_PENDING:
res = self.determine_next_move_state(msg)
if res == DC_CONSTANT_MSG_MOVESTATE_MOVING:
self.schedule_move(msg)
msg.move_state = DC_CONSTANT_MSG_MOVESTATE_MOVING
elif res == DC_CONSTANT_MSG_MOVESTATE_STAY:
self.log("STAY uid=%s message-id=%s" % (msg.uid, message_id))
msg.move_state = DC_CONSTANT_MSG_MOVESTATE_STAY
else:
self.log("PENDING uid=%s message-id=%s in-reply-to=%s" %
(msg.uid, message_id, msg["In-Reply-To"]))
return msg.move_state
def determine_next_move_state(self, msg):
""" Return the next move state for this message.
Only call this function if the message is pending.
This function works with the DB, does not perform any IMAP commands.
"""
self.log("shall_move %s " % (normalized_messageid(msg)))
assert self.foldername in (INBOX, SENT)
assert msg.move_state == DC_CONSTANT_MSG_MOVESTATE_PENDING
if msg.foldername == MVBOX:
self.log("is already in mvbox, next state is STAY %s" %
(normalized_messageid(msg)))
return DC_CONSTANT_MSG_MOVESTATE_STAY
last_dc_count = 0
while 1:
last_dc_count = (last_dc_count + 1) if is_dc_message(msg) else 0
in_reply_to = normalized_messageid(msg.get("In-Reply-To", ""))
if not in_reply_to:
type_msg = "DC" if last_dc_count else "CLEAR"
self.log("detected thread-start %s message" % type_msg,
normalized_messageid(msg))
if last_dc_count > 0:
return DC_CONSTANT_MSG_MOVESTATE_MOVING
else:
return DC_CONSTANT_MSG_MOVESTATE_STAY
newmsg = self.get_message_from_db(in_reply_to)
if not newmsg:
self.log("failed to fetch from db:", in_reply_to)
# we don't have the parent message ... maybe because
# it hasn't arrived (yet), was deleted or we failed to
# scan/fetch it:
if last_dc_count >= 4:
self.log(
"no thread-start found, but last 4 messages were DC")
return DC_CONSTANT_MSG_MOVESTATE_MOVING
else:
self.log("pending: missing parent, last_dc_count=%x" %
(last_dc_count, ))
return DC_CONSTANT_MSG_MOVESTATE_PENDING
elif newmsg.move_state == DC_CONSTANT_MSG_MOVESTATE_MOVING:
self.log("parent was a moved message")
return DC_CONSTANT_MSG_MOVESTATE_MOVING
else:
msg = newmsg
assert 0, "should never arrive here"
def schedule_move(self, msg):
message_id = normalized_messageid(msg)
assert msg.foldername != MVBOX
self.log("scheduling move message-id=%s" % (message_id))
self.pending_imap_jobs = True
def has_message(self, message_id):
assert isinstance(message_id, str)
return message_id in self.db_messages
def get_message_from_db(self, message_id):
return self.db_messages.get(normalized_messageid(message_id))
def store_message(self, message_id, msg):
mid2 = normalized_messageid(msg)
message_id = normalized_messageid(message_id)
assert message_id == mid2
assert message_id not in self.db_messages, message_id
assert msg.foldername in (MVBOX, SENT, INBOX)
self.db_messages[message_id] = msg
self.log("stored new message message-id=%s" % (message_id, ))
def forget_about_too_old_pending_messages(self):
# some housekeeping but not sure if neccessary
# because the involved sql-statements
# probably don't care if there are some foreever-pending messages
now = time.time()
for dbmid, dbmsg in self.db_messages.items():
if dbmsg.move_state == DC_CONSTANT_MSG_MOVESTATE_PENDING:
delay = now - dbmsg.fetch_retrieve_time
if delay > self.pendingtimeout:
dbmsg.move_state = DC_CONSTANT_MSG_MOVESTATE_STAY
self.log("pendingtimeout: message now set to stay", dbmid)
def perform_imap_jobs(self):
with self.wlog("perform_imap_jobs()"):
if self.foldername in (INBOX, SENT):
to_move_uids = []
to_move_msgs = []
# determine all uids of messages that are to be moved
for dbmid, dbmsg in self.db_messages.items():
if dbmsg.move_state == DC_CONSTANT_MSG_MOVESTATE_MOVING:
if dbmsg.uid > 0: # else it's already moved?
to_move_uids.append(dbmsg.uid)
to_move_msgs.append(dbmsg)
if to_move_uids:
self.move(to_move_uids)
# now that we moved let's invalidate "uid" because it's
# not there anyore in thie folder
for dbmsg in to_move_msgs:
dbmsg.uid = 0
self.pending_imap_jobs = False
def _run_in_thread(self):
self.connect()
if self.foldername == INBOX:
# INBOX loop should wait until MVBOX polled once
mvbox.event_initial_polling_complete.wait()
now = time.time()
while True:
self.perform_imap_jobs()
self.perform_imap_fetch()
if self.foldername == MVBOX:
# signal that MVBOX has polled once
self.event_initial_polling_complete.set()
elif self.foldername == INBOX:
# it's not clear we need to do this housekeeping
# (depends on the SQL statements)
self.forget_about_too_old_pending_messages()
self.perform_imap_idle()
def start_thread_loop(self):
assert not self._thread
self._thread = t = threading.Thread(target=self._run_in_thread)
t.start()
by_size = sorted(list(response.viewitems()), key=lambda m: m[1]['RFC822.SIZE'])
sizes = [ m[1]['RFC822.SIZE'] for m in by_size ]
# select msgs that are bigger than threshold
bigger = bisect.bisect_right(sizes, options.threshold) # index of the first message bigger than threshold
big_messages = by_size[bigger:]
print "There are %d messages bigger than %s" % (len(big_messages), options.threshold)
big_uids = [ msg[0] for msg in big_messages ]
def print_messages():
print "\n--- Messages bigger than {}: ".format(options.threshold).ljust(80, '-')
print "Format: <date> | <size> | <from> | <subject>"
for msgid, data in big_messages:
headers = email.message_from_string(data['RFC822.HEADER'])
size = data['RFC822.SIZE']
date = data['INTERNALDATE']
print "{} | {} | {} | {}".format(date, size, headers['from'], headers['subject'])
if not options.no_label:
print "Labeling big messages with label %s" % options.label
server.create_folder(options.label)
server.copy(big_uids, options.label)
print "Your messages larger than {} bytes have been labeled with label {}".format(options.threshold, options.label)
if options.print_msgs or options.no_label:
print_messages()
class DirectImap:
def __init__(self, account: Account) -> None:
self.account = account
self.logid = account.get_config("displayname") or id(account)
self._idling = False
self.connect()
def connect(self):
host = self.account.get_config("configured_mail_server")
port = int(self.account.get_config("configured_mail_port"))
security = int(self.account.get_config("configured_mail_security"))
user = self.account.get_config("addr")
pw = self.account.get_config("mail_pw")
if security == const.DC_SOCKET_PLAIN:
ssl_context = None
else:
ssl_context = ssl.create_default_context()
# don't check if certificate hostname doesn't match target hostname
ssl_context.check_hostname = False
# don't check if the certificate is trusted by a certificate authority
ssl_context.verify_mode = ssl.CERT_NONE
if security == const.DC_SOCKET_STARTTLS:
self.conn = IMAPClient(host, port, ssl=False)
self.conn.starttls(ssl_context)
elif security == const.DC_SOCKET_PLAIN:
self.conn = IMAPClient(host, port, ssl=False)
elif security == const.DC_SOCKET_SSL:
self.conn = IMAPClient(host, port, ssl_context=ssl_context)
self.conn.login(user, pw)
self.select_folder("INBOX")
def shutdown(self):
try:
self.conn.idle_done()
except (OSError, IMAPClientError):
pass
try:
self.conn.logout()
except (OSError, IMAPClientError):
print("Could not logout direct_imap conn")
def create_folder(self, foldername):
try:
self.conn.create_folder(foldername)
except imaplib.IMAP4.error as e:
print("Can't create", foldername, "probably it already exists:", str(e))
def select_folder(self, foldername):
assert not self._idling
return self.conn.select_folder(foldername)
def select_config_folder(self, config_name):
""" Return info about selected folder if it is
configured, otherwise None. """
if "_" not in config_name:
config_name = "configured_{}_folder".format(config_name)
foldername = self.account.get_config(config_name)
if foldername:
return self.select_folder(foldername)
def list_folders(self):
""" return list of all existing folder names"""
assert not self._idling
folders = []
for meta, sep, foldername in self.conn.list_folders():
folders.append(foldername)
return folders
def delete(self, range, expunge=True):
""" delete a range of messages (imap-syntax).
If expunge is true, perform the expunge-operation
to make sure the messages are really gone and not
just flagged as deleted.
"""
self.conn.set_flags(range, [DELETED])
if expunge:
self.conn.expunge()
def get_all_messages(self):
assert not self._idling
# Flush unsolicited responses. IMAPClient has problems
# dealing with them: https://github.com/mjs/imapclient/issues/334
# When this NOOP was introduced, next FETCH returned empty
# result instead of a single message, even though IMAP server
# can only return more untagged responses than required, not
# less.
self.conn.noop()
return self.conn.fetch(ALL, [FLAGS])
def get_unread_messages(self):
assert not self._idling
res = self.conn.fetch(ALL, [FLAGS])
return [uid for uid in res
if SEEN not in res[uid][FLAGS]]
def mark_all_read(self):
messages = self.get_unread_messages()
if messages:
res = self.conn.set_flags(messages, [SEEN])
print("marked seen:", messages, res)
def get_unread_cnt(self):
return len(self.get_unread_messages())
def dump_imap_structures(self, dir, logfile):
assert not self._idling
stream = io.StringIO()
def log(*args, **kwargs):
kwargs["file"] = stream
print(*args, **kwargs)
empty_folders = []
for imapfolder in self.list_folders():
self.select_folder(imapfolder)
messages = list(self.get_all_messages())
if not messages:
empty_folders.append(imapfolder)
continue
log("---------", imapfolder, len(messages), "messages ---------")
# get message content without auto-marking it as seen
# fetching 'RFC822' would mark it as seen.
requested = [b'BODY.PEEK[]', FLAGS]
for uid, data in self.conn.fetch(messages, requested).items():
body_bytes = data[b'BODY[]']
if not body_bytes:
log("Message", uid, "has empty body")
continue
flags = data[FLAGS]
path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
path.mkdir(parents=True, exist_ok=True)
fn = path.joinpath(str(uid))
fn.write_bytes(body_bytes)
log("Message", uid, fn)
email_message = email.message_from_bytes(body_bytes)
log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
if empty_folders:
log("--------- EMPTY FOLDERS:", empty_folders)
print(stream.getvalue(), file=logfile)
def idle_start(self):
""" switch this connection to idle mode. non-blocking. """
assert not self._idling
res = self.conn.idle()
self._idling = True
return res
def idle_check(self, terminate=False):
""" (blocking) wait for next idle message from server. """
assert self._idling
self.account.log("imap-direct: calling idle_check")
res = self.conn.idle_check(timeout=30)
if len(res) == 0:
raise TimeoutError
if terminate:
self.idle_done()
self.account.log("imap-direct: idle_check returned {!r}".format(res))
return res
def idle_wait_for_seen(self):
""" Return first message with SEEN flag
from a running idle-stream REtiurn.
"""
while 1:
for item in self.idle_check():
if item[1] == FETCH:
if item[2][0] == FLAGS:
if SEEN in item[2][1]:
return item[0]
def idle_done(self):
""" send idle-done to server if we are currently in idle mode. """
if self._idling:
res = self.conn.idle_done()
self._idling = False
return res
def append(self, folder, msg):
"""Upload a message to *folder*.
Trailing whitespace or a linebreak at the beginning will be removed automatically.
"""
if msg.startswith("\n"):
msg = msg[1:]
msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
self.conn.append(folder, msg)
def get_uid_by_message_id(self, message_id):
msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
if len(msgs) == 0:
raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
return msgs[0]
class imap:
def __init__(self):
try:
dbfile = open('.credentials', 'rb')
db = pickle.load(dbfile)
temp_server = db["server"]
if (temp_server == ""):
self.server_fail_flag = 1 # declaring that server object of imap is not initialised
pass
else:
try:
self.server = IMAPClient(temp_server, use_uid=True)
self.server_fail_flag = 0
except:
self.server_fail_flag = 1 # declaring that server object of imap is not initialised
print(
"Couldn't able to sign in. Please try again with valied credentials"
)
except:
self.server_fail_flag = 1 # declaring that server object of imap is not initialised
file = open(".credentials", "w+")
file.close()
def login(self, id, passwd, server):
self.server_name = server
self.email = id
self.passwd = passwd
try:
if (self.server_fail_flag == 1):
self.server = IMAPClient(self.server_name, use_uid=True)
self.server.login(self.email, self.passwd)
return 1
print("Login successful")
except:
return 0
def logout(self):
print("Logged out")
self.server.logout()
def list_folders(self):
print("LIST OF FOLDERS")
folders = self.server.list_folders()
for i in folders:
print(i[-1])
return folders
print("")
print(server.get_gmail_labels(messages))
def createFolder(self, folderName):
self.server.create_folder(folderName)
print("Folder named - %s - is created successfully..." % folderName)
def deleteFolder(self, folderName):
self.server.delete_folder(folderName)
def deleteEmail(self, messages):
self.server.delete_messages(messages)
def list_mails(self, folder_name):
select_info = self.server.select_folder(folder_name)
print('%d messages in %s' % (select_info[b'EXISTS'], folder_name))
# messages = server.search(['FROM', '*****@*****.**'])
messages = self.server.search(['ALL'])
# messages = self.server.sort(['ARRIVAL'])
print("%s messages from %s" % (len(messages), folder_name))
fetched_msgs = self.server.fetch(messages, ['ENVELOPE']).items()
return fetched_msgs
def search_mails(self, catagory, parameter):
if (catagory in ["FROM", "TEXT", "SINCE", "SUBJECT"]):
print([catagory, parameter])
messages = self.server.search([catagory, parameter])
print("%s messages from Search catagory -> %s" %
(len(messages), catagory))
fetched_msgs = self.server.fetch(messages, ['ENVELOPE']).items()
print("fetched Messages are ", fetched_msgs)
return fetched_msgs
else:
print([catagory])
messages = self.server.search([catagory])
print(messages)
print("%s messages from Search catagory -> %s" %
(len(messages), catagory))
fetched_msgs = self.server.fetch(messages, ['ENVELOPE']).items()
print("fetched Messages are ", fetched_msgs)
return fetched_msgs
def fetch_email_content(self, msg_id):
for msgid, data in self.server.fetch(msg_id, 'RFC822').items():
# print("\nMESSAGE", msgid)
# print("\nDATA IS => \n", data)
# envelope = data[b'ENVELOPE']
msg = email.message_from_bytes(data[b'RFC822'])
# print(msgid, email_message.get('From'), email_message.get('Subject'))
# print(email_message.get('To'), email_message.get('body'))
subject = msg.get('Subject')
if msg.is_multipart():
# iterate over email parts
for part in msg.walk():
# extract content type of email
content_type = part.get_content_type()
content_disposition = str(part.get("Content-Disposition"))
try:
# get the email body
body = part.get_payload(decode=True).decode()
except:
pass
if content_type == "text/plain" and "attachment" not in content_disposition:
# print text/plain emails and skip attachments
print(body)
elif "attachment" in content_disposition:
# download attachment
filename = part.get_filename()
if filename:
if not os.path.isdir(subject):
# make a folder for this email (named after the subject)
os.mkdir(subject)
filepath = os.path.join(subject, filename)
# download attachment and save it
open(filepath,
"wb").write(part.get_payload(decode=True))
else:
# extract content type of email
content_type = msg.get_content_type()
# get the email body
body = msg.get_payload(decode=True).decode()
if content_type == "text/plain":
# print only text email parts
print(body)
if content_type == "text/html":
# if it's HTML, create a new HTML file and open it in browser
if not os.path.isdir(subject):
# make a folder for this email (named after the subject)
os.mkdir(subject)
filename = "{}.html".format(subject[:50])
filepath = os.path.join(subject, filename)
# write the file
open(filepath, "w").write(body)
# some data about mail
to = msg.get("To")
mail_from = msg.get("From")
mail_date = msg.get("Date")
mail_subject = msg.get("Subject")
print("To =>", to)
print("From =>", mail_from)
print("Date =>", mail_date)
print("=" * 100)
# returning the useful values
return (filepath, body, to, mail_from, mail_date, mail_subject)
class IMAP(Thread):
def __init__(self, filman):
super(IMAP, self).__init__()
self.imap = IMAPClient(HOST, use_uid=True, ssl=ssl)
self.imap.login(USERNAME, PASSWORD)
self.messages = []
self.filterman = filman
self.counter = 0
self.check_dests()
self.loop()
def check_dests(self):
dests = self.filterman.get_dests()
for d in dests:
if not self.imap.folder_exists(d):
self.imap.create_folder(d)
logging.info('[create folder] %s' % d)
if not self.imap.folder_exists(default_not_matched_dest):
self.imap.create_folder(default_not_matched_dest)
logging.info('[create folder] %s' % default_not_matched_dest)
def mark_as_unread(self, msgs):
return self.imap.remove_flags(msgs, ('\\SEEN'))
def check(self):
server = self.imap
select_info = server.select_folder('INBOX')
logging.info("source imap inited: %r" % select_info)
messages = server.search(['NOT SEEN'])
messages = sorted(messages, reverse=True)
self.messages = list(messages)
logging.info('got %d unread messages' % len(self.messages))
def idle(self, secs=30):
server = self.imap
server.idle()
responses = server.idle_check(timeout=secs)
text, responses = server.idle_done()
logging.info('idle response: %s' % (responses))
return not responses
def loop(self):
logging.info('enter loop %d' % self.counter)
self.counter += 1
self.check()
while self.messages:
self._dozen()
self.imap.close_folder()
def _dozen(self):
if self.messages:
msgs = self.messages[:12]
self.messages = self.messages[12:]
else:
return
logging.info('processing the first %d msgs; left %d...' % (
len(msgs), len(self.messages)))
logging.info(msgs)
response = self.imap.fetch(msgs, ['RFC822'])
msgs = [(msgid, Msg(string=data['RFC822']))
for (msgid, data) in response.iteritems()]
self.filterman.test_match_and_take_action(self.imap, msgs)
def run(self):
count = 0
while True:
count += 1
logging.info('idle counter: %d' % count)
self.idle() or self.loop()
sleep(10)
if not count % 5: # do loop every 10 runs.
self.loop()
class IMAPBot(object):
IMAPBotError = IMAPBotError
def __init__(self, host, username, password, ssl=True):
self.server = IMAPClient(host, use_uid=True, ssl=ssl)
self.server.login(username, password)
if 'IDLE' not in self.server.capabilities():
raise IMAPBotError('Sorry, this IMAP server does not support IDLE.')
# the folder where processed emails go
self.processed_folder = 'imapbot_processed'
self.idle_timeout = 5 # seconds
self._is_idle = False
self._run = True
self._create_folder(self.processed_folder)
def check_mail(self):
select_info = self.server.select_folder('INBOX')
print '%d messages in INBOX' % select_info['EXISTS']
messages = self.server.search(['UNSEEN'])
messages = self.server.search(['NOT DELETED'])
print "%d messages that haven't been seen" % len(messages)
if not messages:
return
#response = self.server.fetch(messages, ['FLAGS', 'INTERNALDATE', 'RFC822.SIZE', 'ENVELOPE', 'RFC822.TEXT'])
response = self.server.fetch(messages, ['FLAGS', 'ENVELOPE', 'RFC822.TEXT'])
for message_id, data in response.iteritems():
message = Message(message_id, data['ENVELOPE'], data['RFC822.TEXT'], data['FLAGS'])
self.process(message)
def complete(self, message):
message_ids = [message.id]
self.server.copy(message_ids, self.processed_folder)
self.server.delete_messages(message_ids)
self.server.expunge()
def _create_folder(self, name):
# make sure the folder doesn't already exist
if self.server.folder_exists(name):
return
self.server.create_folder(name)
def handle_message(self, message):
print 'message id: {}, from: {}:'.format(message.id, message.envelope.get_email('from'))
with open('message.txt', 'ab') as fh:
fh.write('{}\n\n'.format(message.text))
print message.plain or message.html or 'no message'
def idle(self):
if self._is_idle:
return
self.server.idle()
self._is_idle = True
return True # this actually changed state
def unidle(self):
if not self._is_idle:
return
self.server.idle_done()
self._is_idle = False
return True # this call actually changed state
def process(self, message):
self.handle_message(message)
self.complete(message)
def run(self):
# process any mail that was in the inbox before coming online
self.check_mail()
# put the connection in idle mode so we get notifications
self.idle()
# loop forever looking for stuff
while self._run:
for message in self.server.idle_check(timeout=self.idle_timeout):
if message[0] == 'OK':
continue
with Unidle(self):
self.check_mail()
def quit(self):
self._run = False
self.unidle()
print self.server.logout()
class IMAP():
"""
Central class for IMAP server communication
"""
Retval = namedtuple('Retval', 'code data')
def __init__(self,
logger,
username,
password,
server='localhost',
port=143,
starttls=False,
imaps=False,
tlsverify=True,
test=False,
timeout=None):
self.logger = logger
self.username = username
self.password = password
self.server = server
self.port = port
self.imaps = imaps
self.starttls = starttls
self.timeout = timeout
self.sslcontext = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
if tlsverify:
self.sslcontext.verify_mode = ssl.CERT_REQUIRED
else:
self.sslcontext.check_hostname = False
self.sslcontext.verify_mode = ssl.CERT_NONE
self.test = test
self.conn = None
def do_select_mailbox(func):
"""
Decorator to do a fresh mailbox SELECT
"""
def wrapper(*args, **kwargs):
if len(args) != 1:
raise AttributeError(
'Size of *args tuple "{0}" isn\'t 1. It looks like you haven\'t specified all '
'method arguments as named arguments!'.format(args))
mailbox = None
for key in ['mailbox', 'source']:
if key in kwargs.keys():
mailbox = kwargs[key]
break
if mailbox is None:
raise KeyError(
'Unable to SELECT a mailbox, kwargs "{0}" doesn\'t contain a mailbox name'
.format(kwargs))
result = args[0].select_mailbox(mailbox)
if not result.code:
raise RuntimeError(result.data)
return func(*args, **kwargs)
return wrapper
def process_error(self, exception, simple_return=False):
"""
Process Python exception by logging a message and optionally showing traceback
"""
trace_info = exc_info()
err_msg = str(exception)
if isinstance(exception, IMAPClient.Error):
err_msg = Helper().byte_to_str(exception.args[0])
self.logger.error("Catching IMAP exception {}: {}".format(
type(exception), err_msg))
if self.logger.isEnabledFor(loglevel_DEBUG):
print_exception(*trace_info)
del trace_info
if simple_return:
return exception
else:
return self.Retval(False, err_msg)
def connect(self, retry=True, logout=False):
"""
Connect to IMAP server and login
"""
if self.starttls:
self.logger.debug(
'Establishing IMAP connection using STARTTLS/{} to {} and logging in with user {}'
.format(self.port, self.server, self.username))
elif self.imaps:
self.logger.debug(
'Establishing IMAP connection using SSL/{} (imaps) to {} and logging in with user {}'
.format(self.port, self.server, self.username))
try:
self.conn = IMAPClient(host=self.server,
port=self.port,
use_uid=True,
ssl=self.imaps,
ssl_context=self.sslcontext,
timeout=self.timeout)
if self.starttls:
self.conn.starttls(ssl_context=self.sslcontext)
login = self.conn.login(self.username, self.password)
login_response = Helper().byte_to_str(login)
# Test login/auth status
login_success = False
noop = self.noop()
if noop.code and noop.data:
login_success = True
if logout:
return self.disconnect()
elif login_success:
return self.Retval(True, login_response)
else:
return self.Retval(False, login_response) # pragma: no cover
except exceptions.LoginError as e:
return self.process_error(e)
except Exception as e:
err_return = self.process_error(e)
if retry:
self.logger.error('Trying one more time to login')
sleep(2)
return self.connect(retry=False, logout=logout)
return err_return
def noop(self):
"""
Do a noop to test login status
"""
try:
noop = self.conn.noop()
noop_response = Helper().byte_to_str(noop[0])
noop_resp_pattern_re = regex_compile('^(Success|NOOP completed)')
login_success = noop_resp_pattern_re.match(noop_response)
return self.Retval(True, login_success)
except IMAPClient.Error as e:
return self.process_error(e)
def disconnect(self):
"""
Disconnect from IMAP server
"""
result = self.conn.logout()
response = Helper().byte_to_str(result)
return self.Retval(response == 'Logging out', response)
def list_mailboxes(self, directory='', pattern='*'):
"""
Get a listing of folders (mailboxes) on the server
"""
try:
raw_list = self.conn.list_folders(directory, pattern)
nice_list = []
for mailbox in raw_list:
flags = []
for flag in mailbox[0]:
flags.append(flag.decode('utf-8'))
nice_list.append({
'name': mailbox[2],
'flags': flags,
'delimiter': mailbox[1].decode("utf-8")
})
return self.Retval(True, nice_list)
except IMAPClient.Error as e:
return self.process_error(e)
def select_mailbox(self, mailbox):
"""
Select a mailbox to work on
"""
self.logger.debug('Switching to mailbox {}'.format(mailbox))
try:
result = self.conn.select_folder(mailbox)
response = {}
for key, value in result.items():
unicode_key = Helper().byte_to_str(key)
if unicode_key == 'FLAGS':
flags = []
for flag in value:
flags.append(Helper().byte_to_str(flag))
response[unicode_key] = tuple(flags)
else:
response[unicode_key] = value
return self.Retval(True, response)
except IMAPClient.Error as e:
return self.process_error(e)
def add_mail(self, mailbox, message, flags=(), msg_time=None):
"""
Add/append a mail to a mailbox
"""
self.logger.debug('Adding a mail into mailbox {}'.format(mailbox))
try:
if not isinstance(message, Mail):
message = Mail(logger=self.logger, mail_native=message)
self.conn.append(mailbox, str(message.get_native()), flags,
msg_time)
# According to rfc4315 we must not return the UID from the response, so we are fetching it ourselves
uids = self.search_mails(mailbox=mailbox,
criteria='HEADER Message-Id "{}"'.format(
message.get_message_id())).data[0]
return self.Retval(True, uids)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def search_mails(self, mailbox, criteria='ALL', autocreate_mailbox=False):
"""
Search for mails in a mailbox
"""
self.logger.debug(
'Searching for mails in mailbox {} and criteria=\'{}\''.format(
mailbox, criteria))
try:
return self.Retval(True, list(self.conn.search(criteria=criteria)))
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def fetch_mails(self, uids, mailbox, return_fields=None):
"""
Retrieve mails from a mailbox
"""
self.logger.debug('Fetching mails with uids {}'.format(uids))
return_raw = True
if return_fields is None:
return_raw = False
return_fields = [b'RFC822']
mails = {}
try:
for uid in uids:
result = self.conn.fetch(uid, return_fields)
if not result:
continue
if return_raw:
mails[uid] = result[uid]
else:
# mails[uid] = Mail(logger=self.logger, uid=uid, mail_native=email.message_from_bytes(result[uid][b'RFC822']))
mails[uid] = Mail(logger=self.logger,
mail_native=email.message_from_bytes(
result[uid][b'RFC822']))
return self.Retval(True, mails)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def get_mailflags(self, uids, mailbox):
"""
Retrieve flags from mails
"""
try:
result = self.conn.get_flags(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to get flags for mail with uid={}: {}'.format(
uid, result))
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def set_mailflags(self, uids, mailbox, flags=[]):
"""
Set and retrieve flags from mails
"""
if self.test:
self.logger.info(
'Would have set mail flags on message uids "{}"'.format(
str(uids)))
return self.Retval(True, None)
else:
self.logger.debug('Setting flags={} on mails uid={}', flags, uids)
try:
result = self.conn.set_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to set and get flags for mail with uid={}: {}'
.format(uid, result))
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def add_mailflags(self, uids, mailbox, flags=[]):
"""
Add and retrieve flags from mails
"""
if self.test:
self.logger.info(
'Would have added mail flags on message uids "{}"'.format(
str(uids)))
return self.Retval(True, None)
else:
self.logger.debug('Adding flags={} on mails uid={}', flags, uids)
try:
result = self.conn.add_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to add and get flags for mail with uid={}: {}'
.format(uid, result))
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def move_mail(self,
message_ids,
source,
destination,
delete_old=True,
expunge=True,
add_flags=None,
set_flags=None):
"""
Move a mail from a mailbox to another
"""
return self.copy_mails(message_ids=message_ids,
source=source,
destination=destination,
delete_old=delete_old,
expunge=expunge,
add_flags=add_flags,
set_flags=set_flags)
@do_select_mailbox
def copy_mails(self,
source,
destination,
message_ids=None,
delete_old=False,
expunge=False,
add_flags=None,
set_flags=None):
"""
Copies one or more mails from a mailbox into another
"""
if self.test:
if delete_old:
self.logger.info(
'Would have moved mail Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode'
.format(message_ids, source, destination))
else:
self.logger.info(
'Would have copied mails with Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode'
.format(message_ids, source, destination))
return self.Retval(True, None)
else:
try:
if delete_old:
self.logger.debug(
'Moving mail Message-Ids="{}" from "{}" to "{}"'.
format(message_ids, source, destination))
else:
self.logger.debug(
'Copying mail Message-Ids="{}" from "{}" to "{}"'.
format(message_ids, source, destination))
# if message_ids is None:
# message_ids = []
# result = self.fetch_mails(uids=uids, mailbox=source)
# if not result.code:
# self.logger.error('Failed to determine Message-Id by uids for mail with uids "{}"', uids)
# return result
# message_ids.append(result.data.keys())
if not self.mailbox_exists(destination).data:
self.logger.info(
'Destination mailbox {} doesn\'t exist, creating it for you'
.format(destination))
result = self.create_mailbox(mailbox=destination)
if not result.code:
self.logger.error(
'Failed to create the mailbox {}: {}'.format(
source, result.data)) # pragma: no cover
return result # pragma: no cover
uids = []
for message_id in message_ids:
result = self.search_mails(
mailbox=source,
criteria='HEADER Message-Id "{}"'.format(message_id))
if not result.code or len(result.data) == 0:
self.logger.error(
'Failed to determine uid by Message-Id for mail with Message-Id "{}"'
.format(message_id))
return self.Retval(False, result.data)
uids.append(result.data[0])
result = self.select_mailbox(source)
if not result.code:
return result # pragma: no cover
self.conn.copy(uids, destination)
if delete_old:
result = self.delete_mails(uids=uids, mailbox=source)
if not result.code:
self.logger.error(
'Failed to remove old mail with Message-Id="{}"/uids="{}": {}'
.format(message_ids, uids,
result.data)) # pragma: no cover
return result # pragma: no cover
if expunge: # TODO don't expunge by default
result = self.expunge(mailbox=source)
if not result.code:
self.logger.error(
'Failed to expunge on mailbox {}: {}'.format(
source, result.data)) # pragma: no cover
return result # pragma: no cover
dest_uids = []
for message_id in message_ids:
result = self.search_mails(
mailbox=destination,
criteria='HEADER Message-Id "{}"'.format(message_id))
if not result.code:
self.logger.error(
'Failed to determine uid by Message-Id for mail with Message-Id "{}"'
.format(message_id)) # pragma: no cover
return result # pragma: no cover
dest_uids.append(result.data[0])
if isinstance(set_flags, list):
self.set_mailflags(uids=dest_uids,
mailbox=destination,
flags=set_flags)
if add_flags:
self.add_mailflags(uids=dest_uids,
mailbox=destination,
flags=add_flags)
return self.Retval(True, dest_uids)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def expunge(self, mailbox):
"""
Expunge mails form a mailbox
"""
self.logger.debug('Expunge mails from mailbox {}'.format(mailbox))
try:
return self.Retval(True, b'Expunge completed.'
in self.conn.expunge())
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
def create_mailbox(self, mailbox):
"""
Create a mailbox
"""
self.logger.debug('Creating mailbox {}'.format(mailbox))
try:
return self.Retval(
True,
self.conn.create_folder(mailbox) == b'Create completed.')
except IMAPClient.Error as e:
return self.process_error(e)
def mailbox_exists(self, mailbox):
"""
Check whether a mailbox exists
"""
try:
return self.Retval(True, self.conn.folder_exists(mailbox))
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
@do_select_mailbox
def delete_mails(self, uids, mailbox):
"""
Delete mails
"""
self.logger.debug('Deleting mails with uid="{}"'.format(uids))
try:
result = self.conn.delete_messages(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error(
'Failed to get flags for mail with uid={} after deleting it: {}'
.format(uid, result))
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
class Hauberk():
def __init__(self, dry_run=False):
self.rules = list()
self.dry = dry_run
def login(self, username, password, server):
logger.info("Connecting to Server")
self.client = IMAPClient(server, use_uid=True)
self.client.login(username, password)
logger.info("Connected")
def add_rule(self, filters, actions):
self.rules.append(Rule(filters=filters, actions=actions))
if not filters:
raise Exception("Must define filters in rule")
if not actions:
raise Exception("Must define actions in rule")
logger.info("Added rule number %s", len(self.rules))
def run(self):
messages = self.client.search()
logger.info("%s messages to process", len(messages))
for msgid, data in self.client.fetch(
messages,
[FetchFlags.ENVELOPE.value, FetchFlags.BODY.value]).items():
envelope = data[FetchFlags.ENVELOPE.value]
body = data[FetchFlags.BODY.value]
message = Message(envelope, body)
logger.debug("Processing message %s: %s", msgid, message.subject)
done = False
for rule in self.rules:
for _filter in rule.filters:
logger.debug("Running filter %s", _filter)
if _filter(message):
logger.debug("Match found!")
# log here that there was a match
for action in rule.actions:
logger.debug("Running action %s", action)
action(self.client, msgid, self.dry)
done = True
break
if done:
break
def select_folder(self, folder):
"""Select working folder.
:param folder: Folder rules should be applied to
"""
logger.info("Running in folder %s", folder)
self.client.select_folder(folder)
def existing_folders(self, folders):
"""Make sure certain folders exist client side.
:param folders: List of folder names
"""
logger.info("Checking for existing folders")
if not isinstance(folders, list):
folders = [folders]
for folder in folders:
logger.debug("Checking for existance of folder %s" % folder)
if not self.client.folder_exists(folder):
logger.warning("Folder %s does not exist, creating" % folder)
if not self.dry:
self.client.create_folder(folder)
class ImapDB(BaseDB):
def __init__(self,
username,
password='******',
host='localhost',
port=143,
*args,
**kwargs):
super().__init__(username, *args, **kwargs)
self.imap = IMAPClient(host, port, use_uid=True, ssl=False)
res = self.imap.login(username, password)
self.cursor.execute(
"SELECT lowModSeq,highModSeq,highModSeqMailbox,highModSeqThread,highModSeqEmail FROM account LIMIT 1"
)
row = self.cursor.fetchone()
self.lastfoldersync = 0
if row:
self.lowModSeq,
self.highModSeq,
self.highModSeqMailbox,
self.highModSeqThread,
self.highModSeqEmail = row
else:
self.lowModSeq = 0
self.highModSeq = 1
self.highModSeqMailbox = 1
self.highModSeqThread = 1
self.highModSeqEmail = 1
# (imapname, readonly)
self.selected_folder = (None, False)
self.mailboxes = {}
self.sync_mailboxes()
self.messages = {}
def get_messages_cached(self, properties=(), id__in=()):
messages = []
if not self.messages:
return messages, id__in, properties
fetch_props = set()
fetch_ids = set(id__in)
for id in id__in:
msg = self.messages.get(id, None)
if msg:
found = True
for prop in properties:
try:
msg[prop]
except (KeyError, AttributeError):
found = False
fetch_props.add(prop)
if found:
fetch_ids.remove(id)
messages.append(msg)
# if one messages is missing, need to fetch all properties
if len(messages) < len(id__in):
fetch_props = properties
return messages, fetch_ids, fetch_props
def get_messages(self,
properties=(),
sort={},
inMailbox=None,
inMailboxOtherThan=(),
id__in=None,
threadId__in=None,
**criteria):
# XXX: id == threadId for now
if id__in is None and threadId__in is not None:
id__in = [id[1:] for id in threadId__in]
if id__in is None:
messages = []
else:
# try get everything from cache
messages, id__in, properties = self.get_messages_cached(
properties, id__in=id__in)
fetch_fields = {
f
for prop, f in FIELDS_MAP.items() if prop in properties
}
if 'RFC822' in fetch_fields:
# remove redundand fields
fetch_fields.discard('RFC822.HEADER')
fetch_fields.discard('RFC822.SIZE')
if inMailbox:
mailbox = self.mailboxes.get(inMailbox, None)
if not mailbox:
raise errors.notFound(f'Mailbox {inMailbox} not found')
mailboxes = [mailbox]
elif inMailboxOtherThan:
mailboxes = [
m for m in self.mailboxes.values()
if m['id'] not in inMailboxOtherThan
]
else:
mailboxes = self.mailboxes.values()
search_criteria = as_imap_search(criteria)
sort_criteria = as_imap_sort(sort) or '' if sort else None
mailbox_uids = {}
if id__in is not None:
if len(id__in) == 0:
return messages # no messages matches empty ids
if not fetch_fields and not sort_criteria:
# when we don't need anything new from IMAP, create empty messages
# useful when requested conditions can be calculated from id (threadId)
messages.extend(
self.messages.get(id, 0) or ImapMessage(id=id)
for id in id__in)
return messages
for id in id__in:
# TODO: check uidvalidity
mailboxid, uidvalidity, uid = parse_message_id(id)
uids = mailbox_uids.get(mailboxid, [])
if not uids:
mailbox_uids[mailboxid] = uids
uids.append(uid)
# filter out unnecessary mailboxes
mailboxes = [m for m in mailboxes if m['id'] in mailbox_uids]
for mailbox in mailboxes:
imapname = mailbox['imapname']
if self.selected_folder[0] != imapname:
self.imap.select_folder(imapname, readonly=True)
self.selected_folder = (imapname, True)
uids = mailbox_uids.get(mailbox['id'], None)
# uids are now None or not empty
# fetch all
if sort_criteria:
if uids:
search = f'{",".join(map(str, uids))} {search_criteria}'
else:
search = search_criteria or 'ALL'
uids = self.imap.sort(sort_criteria, search)
elif search_criteria:
if uids:
search = f'{",".join(map(str, uids))} {search_criteria}'
uids = self.imap.search(search)
if uids is None:
uids = '1:*'
fetch_fields.add('UID')
fetches = self.imap.fetch(uids, fetch_fields)
for uid, data in fetches.items():
id = format_message_id(mailbox['id'], mailbox['uidvalidity'],
uid)
msg = self.messages.get(id, None)
if not msg:
msg = ImapMessage(id=id, mailboxIds=[mailbox['id']])
self.messages[id] = msg
for k, v in data.items():
msg[k.decode()] = v
messages.append(msg)
return messages
def changed_record(self, ifolderid, uid, flags=(), labels=()):
res = self.dmaybeupdate(
'imessages', {
'flags': json.dumps(sorted(flags)),
'labels': json.dumps(sorted(labels)),
}, {
'ifolderid': ifolderid,
'uid': uid
})
if res:
msgid = self.dgefield('imessages', {
'ifolderid': ifolderid,
'uid': uid
}, 'msgid')
self.mark_sync(msgid)
def import_message(self, rfc822, mailboxIds, keywords):
folderdata = self.dget('ifolders')
foldermap = {f['ifolderid']: f for f in folderdata}
jmailmap = {
f['jmailboxid']: f
for f in folderdata if f.get('jmailboxid', False)
}
# store to the first named folder - we can use labels on gmail to add to other folders later.
id, others = mailboxIds
imapname = jmailmap[id][imapname]
flags = set(keywords)
for kw in flags:
if kw in KEYWORD2FLAG:
flags.remove(kw)
flags.add(KEYWORD2FLAG[kw])
appendres = self.imap.append('imapname', '(' + ' '.join(flags) + ')',
datetime.now(), rfc822)
# TODO: compare appendres[2] with uidvalidity
uid = appendres[3]
fdata = jmailmap[mailboxIds[0]]
self.do_folder(fdata['ifolderid'], fdata['label'])
ifolderid = fdata['ifolderid']
msgdata = self.dgetone('imessages', {
'ifolderid': ifolderid,
'uid': uid,
}, 'msgid,thrid,size')
# XXX - did we fail to sync this back? Annoying
if not msgdata:
raise Exception(
'Failed to get back stored message from imap server')
# save us having to download it again - drop out of transaction so we don't wait on the parse
message = parse.parse(rfc822, msgdata['msgid'])
self.begin()
self.dinsert(
'jrawmessage', {
'msgid': msgdata['msgid'],
'parsed': json.dumps('message'),
'hasAttachment': message['hasattachment'],
})
self.commit()
return msgdata
def update_messages(self, changes, idmap):
if not changes:
return {}, {}
changed = {}
notchanged = {}
map = {}
msgids = set(changes.keys())
sql = 'SELECT msgid,ifolderid,uid FROM imessages WHERE msgid IN (' + (
('?,' * len(msgids))[:-1]) + ')'
self.cursor.execute(sql, list(msgids))
for msgid, ifolderid, uid in self.cursor:
if not msgid in map:
map[msgid] = {ifolderid: {uid}}
elif not ifolderid in map[msgid]:
map[msgid][ifolderid] = {uid}
else:
map[msgid][ifolderid].add(uid)
msgids.discard(msgid)
for msgid in msgids:
notchanged[msgid] = {
'type': 'notFound',
'description': 'No such message on server',
}
folderdata = self.dget('ifolders')
foldermap = {f['ifolderid']: f for f in folderdata}
jmailmap = {
f['jmailboxid']: f
for f in folderdata if 'jmailboxid' in f
}
jmapdata = self.dget('jmailboxes')
jidmap = {d['jmailboxid']: (d['role'] or '') for d in jmapdata}
jrolemap = {
d['role']: d['jmailboxid']
for d in jmapdata if 'role' in d
}
for msgid in map.keys():
action = changes[msgid]
try:
for ifolderid, uids in map[msgid].items():
# TODO: merge similar actions?
imapname = foldermap[ifolderid]['imapname']
uidvalidity = foldermap[ifolderid]['uidvalidity']
if self.selected_folder != (imapname, False):
self.imap.select_folder(imapname)
self.selected_folder = (imapname, False)
if imapname and uidvalidity and 'keywords' in action:
flags = set(action['keywords'])
for kw in flags:
if kw in KEYWORD2FLAG:
flags.remove(kw)
flags.add(KEYWORD2FLAG[kw])
self.imap.set_flags(uids, flags, silent=True)
if 'mailboxIds' in action:
mboxes = [idmap(k) for k in action['mailboxIds'].keys()]
# existing ifolderids containing this message
# identify a source message to work from
ifolderid = sorted(map[msgid])[0]
uid = sorted(map[msgid][ifolderid])[0]
imapname = foldermap[ifolderid]['imapname']
uidvalidity = foldermap[ifolderid]['uidvalidity']
# existing ifolderids with this message
current = set(map[msgid].keys())
# new ifolderids that should contain this message
new = set(jmailmap[x]['ifolderid'] for x in mboxes)
for ifolderid in new:
# unless there's already a matching message in it
if current.pop(ifolderid):
continue
# copy from the existing message
newfolder = foldermap[ifolderid]['imapname']
self.imap.copy(imapname, uidvalidity, uid, newfolder)
for ifolderid in current:
# these ifolderids didn't exist in new, so delete all matching UIDs from these folders
self.imap.move(
foldermap[ifolderid]['imapname'],
foldermap[ifolderid]['uidvalidity'],
map[msgid][ifolderid], # uids
)
except Exception as e:
notchanged[msgid] = {'type': 'error', 'description': str(e)}
raise e
else:
changed[msgid] = None
return changed, notchanged
def destroy_messages(self, ids):
if not ids:
return [], {}
destroymap = defaultdict(dict)
notdestroyed = {}
idset = set(ids)
rows = self.dget('imessages', {'msgid': ('IN', idset)},
'msgid,ifolderid,uid')
for msgid, ifolderid, uid in rows:
idset.discard(msgid)
destroymap[ifolderid][uid] = msgid
for msgid in idset:
notdestroyed[msgid] = {
'type': 'notFound',
'description': "No such message on server",
}
folderdata = self.dget('ifolders')
foldermap = {d['ifolderid']: d for d in folderdata}
jmailmap = {
d['jmailboxid']: d
for d in folderdata if 'jmailboxid' in d
}
destroyed = []
for ifolderid, ifolder in destroymap.items():
#TODO: merge similar actions?
if not ifolder['imapname']:
for msgid in destroymap[ifolderid]:
notdestroyed[msgid] = \
{'type': 'notFound', 'description': "No folder"}
self.imap.move(ifolder['imapname'], ifolder['uidvalidity'],
destroymap[ifolderid].keys(), None)
destroyed.extend(destroymap[ifolderid].values())
return destroyed, notdestroyed
def deleted_record(self, ifolderid, uid):
msgid = self.dgetfield('imessages', {
'ifolderid': ifolderid,
'uid': uid
}, 'msgid')
if msgid:
self.ddelete('imessages', {'ifolderid': ifolderid, 'uid': uid})
self.mark_sync(msgid)
def get_raw_message(self, msgid, part=None):
self.cursor.execute(
'SELECT imapname,uidvalidity,uid FROM ifolders JOIN imessages USING (ifolderid) WHERE msgid=?',
[msgid])
imapname, uidvalidity, uid = self.cursor.fetchone()
if not imapname:
return None
typ = 'message/rfc822'
if part:
parsed = self.fill_messages([msgid])
typ = find_type(parsed[msgid], part)
res = self.imap.getpart(imapname, uidvalidity, uid, part)
return typ, res['data']
def get_mailboxes(self, fields=None, **criteria):
byimapname = {}
# TODO: LIST "" % RETURN (STATUS (UNSEEN MESSAGES HIGHESTMODSEQ MAILBOXID))
for flags, sep, imapname in self.imap.list_folders():
status = self.imap.folder_status(imapname, ([
'MESSAGES', 'UIDVALIDITY', 'UIDNEXT', 'HIGHESTMODSEQ', 'X-GUID'
]))
flags = [f.lower() for f in flags]
roles = [f for f in flags if f not in KNOWN_SPECIALS]
label = roles[0].decode() if roles else imapname
role = ROLE_MAP.get(label.lower(), None)
can_select = b'\\noselect' not in flags
byimapname[imapname] = {
# Dovecot can fetch X-GUID
'id': status[b'X-GUID'].decode(),
'parentId': None,
'name': imapname,
'role': role,
'sortOrder': 2 if role else (1 if role == 'inbox' else 3),
'isSubscribed': True, # TODO: use LSUB
'totalEmails': status[b'MESSAGES'],
'unreadEmails': 0,
'totalThreads': 0,
'unreadThreads': 0,
'myRights': {
'mayReadItems': can_select,
'mayAddItems': can_select,
'mayRemoveItems': can_select,
'maySetSeen': can_select,
'maySetKeywords': can_select,
'mayCreateChild': True,
'mayRename': False if role else True,
'mayDelete': False if role else True,
'maySubmit': can_select,
},
'imapname': imapname,
'sep': sep.decode(),
'uidvalidity': status[b'UIDVALIDITY'],
'uidnext': status[b'UIDNEXT'],
# Data sync properties
'createdModSeq': status[b'UIDVALIDITY'], # TODO: persist
'updatedModSeq':
status[b'UIDVALIDITY'], # TODO: from persistent storage
'updatedNotCountsModSeq':
status[b'UIDVALIDITY'], # TODO: from persistent storage
'emailHighestModSeq': status[b'HIGHESTMODSEQ'],
'deleted': 0,
}
# set name and parentId for child folders
for imapname, mailbox in byimapname.items():
names = imapname.rsplit(mailbox['sep'], maxsplit=1)
if len(names) == 2:
mailbox['parentId'] = byimapname[names[0]]['id']
mailbox['name'] = names[1]
# update cache
self.mailboxes = {mbox['id']: mbox for mbox in byimapname.values()}
return byimapname.values()
def sync_mailboxes(self):
self.get_mailboxes()
def mailbox_imapname(self, parentId, name):
parent = self.mailboxes.get(parentId, None)
if not parent:
raise errors.notFound('parent folder not found')
return parent['imapname'] + parent['sep'] + name
def create_mailbox(self,
name=None,
parentId=None,
isSubscribed=True,
**kwargs):
if not name:
raise errors.invalidProperties('name is required')
imapname = self.mailbox_imapname(parentId, name)
# TODO: parse returned MAILBOXID
try:
res = self.imap.create_folder(imapname)
except IMAPClientError as e:
desc = str(e)
if '[ALREADYEXISTS]' in desc:
raise errors.invalidArguments(desc)
except Exception:
raise errors.serverFail(res.decode())
if not isSubscribed:
self.imap.unsubscribe_folder(imapname)
status = self.imap.folder_status(imapname, ['UIDVALIDITY'])
self.sync_mailboxes()
return f"f{status[b'UIDVALIDITY']}"
def update_mailbox(self,
id,
name=None,
parentId=None,
isSubscribed=None,
sortOrder=None,
**update):
mailbox = self.mailboxes.get(id, None)
if not mailbox:
raise errors.notFound('mailbox not found')
imapname = mailbox['imapname']
if (name is not None and name != mailbox['name']) or \
(parentId is not None and parentId != mailbox['parentId']):
if not name:
raise errors.invalidProperties('name is required')
newimapname = self.mailbox_imapname(parentId, name)
res = self.imap.rename_folder(imapname, newimapname)
if b'NO' in res or b'BAD' in res:
raise errors.serverFail(res.encode())
if isSubscribed is not None and isSubscribed != mailbox['isSubscribed']:
if isSubscribed:
res = self.imap.subscribe_folder(imapname)
else:
res = self.imap.unsubscribe_folder(imapname)
if b'NO' in res or b'BAD' in res:
raise errors.serverFail(res.encode())
if sortOrder is not None and sortOrder != mailbox['sortOrder']:
# TODO: update in persistent storage
mailbox['sortOrder'] = sortOrder
self.sync_mailboxes()
def destroy_mailbox(self, id):
mailbox = self.mailboxes.get(id, None)
if not mailbox:
raise errors.notFound('mailbox not found')
res = self.imap.delete_folder(mailbox['imapname'])
if b'NO' in res or b'BAD' in res:
raise errors.serverFail(res.encode())
mailbox['deleted'] = datetime.now().timestamp()
self.sync_mailboxes()
def create_submission(self, new, idmap):
if not new:
return {}, {}
todo = {}
createmap = {}
notcreated = {}
for cid, sub in new.items():
msgid = idmap.get(sub['emailId'], sub['emailId'])
if not msgid:
notcreated[cid] = {'error': 'nos msgid provided'}
continue
thrid = self.dgetfield('jmessages', {
'msgid': msgid,
'deleted': 0
}, 'thrid')
if not thrid:
notcreated[cid] = {'error': 'message does not exist'}
continue
id = self.dmake(
'jsubmission', {
'sendat':
datetime.fromisoformat()(sub['sendAt']).isoformat()
if sub['sendAt'] else datetime.now().isoformat(),
'msgid':
msgid,
'thrid':
thrid,
'envelope':
json.dumps(sub['envelope']) if 'envelope' in sub else None,
})
createmap[cid] = {'id': id}
todo[cid] = msgid
self.commit()
for cid, sub in todo.items():
type, rfc822 = self.get_raw_message(todo[cid])
self.imap.send_mail(rfc822, sub['envelope'])
return createmap, notcreated
def update_submission(self, changed, idmap):
return {}, {x: 'change not supported' for x in changed.keys()}
def destroy_submission(self, destroy):
if not destroy:
return [], {}
destroyed = []
notdestroyed = {}
namemap = {}
for subid in destroy:
deleted = self.dgetfield('jsubmission', {'jsubid': subid},
'deleted')
if deleted:
destroy.append(subid)
self.ddelete('jsubmission', {'jsubid': subid})
else:
notdestroyed[subid] = {
'type': 'notFound',
'description': 'submission not found'
}
self.commit()
return destroyed, notdestroyed
def _initdb(self):
super()._initdb()
self.dbh.execute("""
CREATE TABLE IF NOT EXISTS ifolders (
ifolderid INTEGER PRIMARY KEY NOT NULL,
jmailboxid INTEGER,
sep TEXT NOT NULL,
imapname TEXT NOT NULL,
label TEXT,
uidvalidity INTEGER,
uidfirst INTEGER,
uidnext INTEGER,
highestmodseq INTEGER,
uniqueid TEXT,
mtime DATE NOT NULL
)""")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS ifolderj ON ifolders (jmailboxid)")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS ifolderlabel ON ifolders (label)")
self.dbh.execute("""
CREATE TABLE IF NOT EXISTS imessages (
imessageid INTEGER PRIMARY KEY NOT NULL,
ifolderid INTEGER,
uid INTEGER,
internaldate DATE,
modseq INTEGER,
flags TEXT,
labels TEXT,
thrid TEXT,
msgid TEXT,
envelope TEXT,
bodystructure TEXT,
size INTEGER,
mtime DATE NOT NULL
)""")
self.dbh.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS imsgfrom ON imessages (ifolderid, uid)"
)
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS imessageid ON imessages (msgid)")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS imessagethrid ON imessages (thrid)")
self.dbh.execute("""
CREATE TABLE IF NOT EXISTS ithread (
messageid TEXT PRIMARY KEY,
sortsubject TEXT,
thrid TEXT
)""")
self.dbh.execute(
"CREATE INDEX IF NOT EXISTS ithrid ON ithread (thrid)")
class IMAP():
"""
Central class for IMAP server communication
"""
Retval = namedtuple('Retval', 'code data')
def __init__(self, logger, username, password,
server='localhost',
port=143,
starttls=False,
imaps=False,
tlsverify=True,
test=False,
timeout=None):
self.logger = logger
self.username = username
self.password = password
self.server = server
self.port = port
self.imaps = imaps
self.starttls = starttls
self.timeout = timeout
self.sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) # TODO add proto arg
if tlsverify:
self.sslcontext.verify_mode = ssl.CERT_REQUIRED
else:
self.sslcontext.verify_mode = ssl.CERT_NONE # TODO improve?
self.test = test
self.conn = None
def do_select_mailbox(func):
"""
Decorator to do a fresh mailbox SELECT
"""
def wrapper(*args, **kwargs):
if len(args) != 1:
raise AttributeError(
'Size of *args tuple "{0}" isn\'t 1. It looks like you haven\'t specified all '
'method arguments as named arguments!'.format(
args))
mailbox = None
for key in ['mailbox', 'source']:
if key in kwargs.keys():
mailbox = kwargs[key]
break
if mailbox is None:
raise KeyError('Unable to SELECT a mailbox, kwargs "{0}" doesn\'t contain a mailbox name'.format(kwargs))
result = args[0].select_mailbox(mailbox)
if not result.code:
raise RuntimeError(result.data)
return func(*args, **kwargs)
return wrapper
def process_error(self, exception, simple_return=False):
"""
Process Python exception by logging a message and optionally showing traceback
"""
trace_info = exc_info()
err_msg = str(exception)
if isinstance(exception, IMAPClient.Error):
err_msg = Helper().byte_to_str(exception.args[0])
self.logger.error('Catching IMAP exception %s: %s', type(exception), err_msg)
if self.logger.isEnabledFor(loglevel_DEBUG):
print_exception(*trace_info)
del trace_info
if simple_return:
return exception
else:
return self.Retval(False, err_msg)
def connect(self, retry=True, logout=False):
"""
Connect to IMAP server and login
"""
if self.starttls:
self.logger.debug('Establishing IMAP connection using STARTTLS/%s to %s and logging in with user %s', self.port, self.server,
self.username)
elif self.imaps:
self.logger.debug('Establishing IMAP connection using SSL/%s (imaps) to %s and logging in with user %s', self.port, self.server,
self.username)
login = ''
err_return = None
try:
self.conn = IMAPClient(host=self.server,
port=self.port,
use_uid=True,
ssl=self.imaps,
ssl_context=self.sslcontext,
timeout=self.timeout)
if self.starttls:
self.conn.starttls(ssl_context=self.sslcontext)
login = self.conn.login(self.username, self.password)
login_response = Helper().byte_to_str(login)
# Test login/auth status
login_success = False
noop = self.noop()
if noop.code and noop.data:
login_success = True
if logout:
return self.disconnect()
elif login_success:
return self.Retval(True, login_response)
else:
return self.Retval(False, login_response) # pragma: no cover
except Exception as e:
err_return = self.process_error(e)
if err_return.data == '[AUTHENTICATIONFAILED] Authentication failed.':
return err_return
if retry:
self.logger.error('Trying one more time to login')
sleep(2)
return self.connect(retry=False, logout=logout)
return err_return
def noop(self):
"""
Do a noop to test login status
"""
try:
noop = self.conn.noop()
noop_response = Helper().byte_to_str(noop[0])
noop_resp_pattern_re = regex_compile('^(Success|NOOP completed)')
login_success = noop_resp_pattern_re.match(noop_response)
return self.Retval(True, login_success)
except IMAPClient.Error as e:
return self.process_error(e)
def disconnect(self):
"""
Disconnect from IMAP server
"""
result = self.conn.logout()
response = Helper().byte_to_str(result)
return self.Retval(response == 'Logging out', response)
def list_mailboxes(self, directory='', pattern='*'):
"""
Get a listing of folders (mailboxes) on the server
"""
try:
raw_list = self.conn.list_folders(directory, pattern)
nice_list = []
for mailbox in raw_list:
flags = []
for flag in mailbox[0]:
flags.append(flag.decode('utf-8'))
nice_list.append({'name': mailbox[2], 'flags': flags, 'delimiter': mailbox[1].decode("utf-8")})
return self.Retval(True, nice_list)
except IMAPClient.Error as e:
return self.process_error(e)
def select_mailbox(self, mailbox):
"""
Select a mailbox to work on
"""
self.logger.debug('Switching to mailbox %s', mailbox)
try:
result = self.conn.select_folder(mailbox)
response = {}
for key, value in result.items():
unicode_key = Helper().byte_to_str(key)
if unicode_key == 'FLAGS':
flags = []
for flag in value:
flags.append(Helper().byte_to_str(flag))
response[unicode_key] = tuple(flags)
else:
response[unicode_key] = value
return self.Retval(True, response)
except IMAPClient.Error as e:
return self.process_error(e)
def add_mail(self, mailbox, message, flags=(), msg_time=None):
"""
Add/append a mail to a mailbox
"""
self.logger.debug('Adding a mail into mailbox %s', mailbox)
try:
if not isinstance(message, Mail):
message = Mail(logger=self.logger, mail_native=message)
message_native = message.get_native()
#self.conn.append(mailbox, message, flags, msg_time)
self._append(mailbox, str(message_native), flags, msg_time)
# According to rfc4315 we must not return the UID from the response, so we are fetching it ourselves
uids = self.search_mails(mailbox=mailbox, criteria='HEADER Message-Id "{0}"'.format(message.get_header('Message-Id'))).data[0]
return self.Retval(True, uids)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def search_mails(self, mailbox, criteria='ALL', autocreate_mailbox=False):
"""
Search for mails in a mailbox
"""
self.logger.debug('Searching for mails in mailbox %s and criteria=\'%s\'', mailbox, criteria)
try:
return self.Retval(True, list(self.conn.search(criteria=criteria)))
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def fetch_mails(self, uids, mailbox, return_fields=None):
"""
Retrieve mails from a mailbox
"""
self.logger.debug('Fetching mails with uids %s', uids)
return_raw = True
if return_fields is None:
return_raw = False
return_fields = [b'RFC822']
mails = {}
try:
for uid in uids:
result = self.conn.fetch(uid, return_fields)
if not result:
continue
if return_raw:
mails[uid] = result[uid]
else:
#mails[uid] = Mail(logger=self.logger, uid=uid, mail_native=email.message_from_bytes(result[uid][b'RFC822']))
mails[uid] = Mail(logger=self.logger, mail_native=email.message_from_bytes(result[uid][b'RFC822']))
return self.Retval(True, mails)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def get_mailflags(self, uids, mailbox):
"""
Retrieve flags from mails
"""
try:
result = self.conn.get_flags(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error('Failed to get flags for mail with uid=%s: %s', uid, result)
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def set_mailflags(self, uids, mailbox, flags=[]):
"""
Set and retrieve flags from mails
"""
if self.test:
self.logger.info('Would have set mail flags on message uids "%s"', str(uids))
return self.Retval(True, None)
else:
self.logger.debug('Setting flags=%s on mails uid=%s', flags, uids)
try:
result = self.conn.set_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error('Failed to set and get flags for mail with uid=%s: %s', uid, result)
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def add_mailflags(self, uids, mailbox, flags=[]):
"""
Add and retrieve flags from mails
"""
if self.test:
self.logger.info('Would have added mail flags on message uids "%s"', str(uids))
return self.Retval(True, None)
else:
self.logger.debug('Adding flags=%s on mails uid=%s', flags, uids)
try:
result = self.conn.add_flags(uids, flags)
_flags = {}
for uid in uids:
_flags[uid] = []
if uid not in result.keys():
self.logger.error('Failed to add and get flags for mail with uid=%s: %s', uid, result)
return self.Retval(False, None)
for flag in result[uid]:
_flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, _flags)
except IMAPClient.Error as e:
return self.process_error(e)
@do_select_mailbox
def move_mail(self, message_ids, source, destination, delete_old=True, expunge=True, add_flags=None, set_flags=None):
"""
Move a mail from a mailbox to another
"""
return self.copy_mails(message_ids=message_ids,
source=source,
destination=destination,
delete_old=delete_old,
expunge=expunge,
add_flags=add_flags,
set_flags=set_flags)
@do_select_mailbox
def copy_mails(self, source, destination, message_ids=None, delete_old=False, expunge=False, add_flags=None, set_flags=None):
"""
Copies one or more mails from a mailbox into another
"""
if self.test:
if delete_old:
self.logger.info('Would have moved mail Message-Ids="%s" from "%s" to "%s", skipping because of beeing in testmode',
message_ids, source, destination)
else:
self.logger.info('Would have copied mails with Message-Ids="%s" from "%s" to "%s", skipping because of beeing in testmode',
message_ids, source, destination)
return self.Retval(True, None)
else:
try:
if delete_old:
self.logger.debug('Moving mail Message-Ids="%s" from "%s" to "%s"', message_ids, source, destination)
else:
self.logger.debug('Copying mail Message-Ids="%s" from "%s" to "%s"', message_ids, source, destination)
#if message_ids is None:
# message_ids = []
# result = self.fetch_mails(uids=uids, mailbox=source)
# if not result.code:
# self.logger.error('Failed to determine Message-Id by uids for mail with uids "%s"', uids)
# return result
# message_ids.append(result.data.keys())
if not self.mailbox_exists(destination).data:
self.logger.info('Destination mailbox %s doesn\'t exist, creating it for you', destination)
result = self.create_mailbox(mailbox=destination)
if not result.code:
self.logger.error('Failed to create the mailbox %s: %s', source, result.data) # pragma: no cover
return result # pragma: no cover
uids = []
for message_id in message_ids:
result = self.search_mails(mailbox=source, criteria='HEADER Message-Id "{0}"'.format(message_id))
if not result.code or len(result.data) == 0:
self.logger.error('Failed to determine uid by Message-Id for mail with Message-Id "%s"', message_id)
return self.Retval(False, result.data)
uids.append(result.data[0])
result = self.select_mailbox(source)
if not result.code:
return result # pragma: no cover
self.conn.copy(uids, destination)
if delete_old:
result = self.delete_mails(uids=uids, mailbox=source)
if not result.code:
self.logger.error('Failed to remove old mail with Message-Id="%s"/uids="%s": %s', message_ids, uids,
result.data) # pragma: no cover
return result # pragma: no cover
if expunge: # TODO don't expunge by default
result = self.expunge(mailbox=source)
if not result.code:
self.logger.error('Failed to expunge on mailbox %s: %s', source, result.data) # pragma: no cover
return result # pragma: no cover
dest_uids = []
for message_id in message_ids:
result = self.search_mails(mailbox=destination, criteria='HEADER Message-Id "{0}"'.format(message_id))
if not result.code:
self.logger.error('Failed to determine uid by Message-Id for mail with Message-Id "%s"',
message_id) # pragma: no cover
return result # pragma: no cover
dest_uids.append(result.data[0])
if isinstance(set_flags, list):
self.set_mailflags(uids=dest_uids, mailbox=destination, flags=set_flags)
if add_flags:
self.add_mailflags(uids=dest_uids, mailbox=destination, flags=add_flags)
return self.Retval(True, dest_uids)
except IMAPClient.Error as e:
return self.process_error(e)
def _append(self, folder, msg, flags=(), msg_time=None): # TODO
"""
FORKED FORM IMAPCLIENT
"""
if msg_time:
if not msg_time.tzinfo: # pragma: no cover
msg_time = msg_time.replace(tzinfo=FixedOffset.for_system()) # pragma: no cover
time_val = '"{0}"'.format(msg_time.strftime("%d-%b-%Y %H:%M:%S %z"))
time_val = imapclient.imapclient.to_unicode(time_val)
else:
time_val = None
return self.conn._command_and_check('append', self.conn._normalise_folder(folder), imapclient.imapclient.seq_to_parenstr(flags),
time_val, Helper.str_to_bytes(msg),
unpack=True)
@do_select_mailbox
def expunge(self, mailbox):
"""
Expunge mails form a mailbox
"""
self.logger.debug('Expunge mails from mailbox %s', mailbox)
try:
return self.Retval(True, b'Expunge completed.' in self.conn.expunge())
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
def create_mailbox(self, mailbox):
"""
Create a mailbox
"""
self.logger.debug('Creating mailbox %s', mailbox)
try:
return self.Retval(True, self.conn.create_folder(mailbox) == b'Create completed.')
except IMAPClient.Error as e:
return self.process_error(e)
def mailbox_exists(self, mailbox):
"""
Check whether a mailbox exists
"""
try:
return self.Retval(True, self.conn.folder_exists(mailbox))
except IMAPClient.Error as e: # pragma: no cover
return self.process_error(e) # pragma: no cover
@do_select_mailbox
def delete_mails(self, uids, mailbox):
"""
Delete mails
"""
self.logger.debug('Deleting mails with uid="%s"', uids)
try:
result = self.conn.delete_messages(uids)
flags = {}
for uid in uids:
flags[uid] = []
if uid not in result.keys():
self.logger.error('Failed to get flags for mail with uid=%s after deleting it: %s', uid, result)
return self.Retval(False, None)
for flag in result[uid]:
flags[uid].append(flag.decode('utf-8'))
return self.Retval(True, flags)
except IMAPClient.Error as e:
return self.process_error(e)
class MailFetcher(object):
def __init__(self, host, user, password, folder_inbox, folder_archive,
folder_error):
self.server = IMAPClient(host)
self.server.login(user, password)
self.folder_inbox = folder_inbox
self.folder_archive = folder_archive
self.folder_error = folder_error
if self.server.folder_exists(self.folder_archive):
logger.info("Archive folder '{folder}' exists.".format(
folder=self.folder_archive))
else:
self.server.create_folder(self.folder_archive)
logger.info("Archive folder '{folder}' created.".format(
folder=self.folder_archive))
if self.server.folder_exists(self.folder_error):
logger.info("Error folder '{folder}' exists.".format(
folder=self.folder_error))
else:
self.server.create_folder(self.folder_error)
logger.info("Error folder '{folder}' created.".format(
folder=self.folder_error))
def findIcs(self, rawmail):
email_message = email.message_from_bytes(rawmail)
for part in email_message.walk():
logger.debug(part.get_content_type())
if ("text/calendar" in part.get_content_type()):
logger.info(
"Found calendar event in mail from {sender} with subject '{subject}'."
.format(sender=email_message.get('From'),
subject=email_message.get('Subject')))
return ical_tools.parse(part.get_payload(decode=True))
def getIcs(self):
select_info = self.server.select_folder(self.folder_inbox)
logger.debug("{num_mails} messages in folder {folder}.".format(
num_mails=select_info[b'EXISTS'], folder=self.folder_inbox))
ics_list = []
messages = self.server.search('ALL')
for message_uid, message_data in self.server.fetch(messages,
'RFC822').items():
logger.debug("Found message with id {id}".format(id=message_uid))
ics = self.findIcs(message_data[b'RFC822'])
if (ics):
ics_list.append((message_uid, ics))
return ics_list
def archiveMessage(self, uid, error=False):
folder = self.folder_archive
if error:
folder = self.folder_error
self.server.move([uid], folder)
logger.info("Message with UID {uid} moved to folder {folder}".format(
uid=uid, folder=folder))
class EmailListener:
"""EmailListener object for listening to an email folder and processing emails.
Attributes:
email (str): The email to listen to.
app_password (str): The password for the email.
folder (str): The email folder to listen in.
attachment_dir (str): The file path to the folder to save scraped
emails and attachments to.
server (IMAPClient): The IMAP server to log into. Defaults to None.
"""
def __init__(self, email, app_password, folder, attachment_dir):
"""Initialize an EmailListener instance.
Args:
email (str): The email to listen to.
app_password (str): The password for the email.
folder (str): The email folder to listen in.
attachment_dir (str): The file path to folder to save scraped
emails and attachments to.
Returns:
None
"""
self.email = email
self.app_password = app_password
self.folder = folder
self.attachment_dir = attachment_dir
self.server = None
def login(self):
"""Logs in the EmailListener to the IMAP server.
Args:
None
Returns:
None
"""
self.server = IMAPClient('imap.gmail.com')
self.server.login(self.email, self.app_password)
self.server.select_folder(self.folder, readonly=False)
def logout(self):
"""Logs out the EmailListener from the IMAP server.
Args:
None
Returns:
None
"""
self.server.logout()
self.server = None
def scrape(self, move=None, unread=False, delete=False):
"""Scrape unread emails from the current folder.
Args:
move (str): The folder to move the emails to. If None, the emails
are not moved. Defaults to None.
unread (bool): Whether the emails should be marked as unread.
Defaults to False.
delete (bool): Whether the emails should be deleted. Defaults to
False.
Returns:
A list of the file paths to each scraped email.
"""
# Ensure server is connected
if type(self.server) is not IMAPClient:
raise ValueError("server attribute must be type IMAPClient")
# List containing the file paths of each file created for an email message
msg_dict = {}
# Search for unseen messages
messages = self.server.search("UNSEEN")
# For each unseen message
for uid, message_data in self.server.fetch(messages, 'RFC822').items():
# Get the message
email_message = email.message_from_bytes(message_data[b'RFC822'])
# Get who the message is from
from_email = self.__get_from(email_message)
# Generate the dict key for this email
key = "{}".format(uid)
# Generate the value dictionary to be filled later
val_dict = {}
email_directory = os.path.join(self.attachment_dir, from_email,
str(uid))
val_dict["email_directory"] = email_directory
val_dict["from_email"] = from_email
if not os.path.exists(email_directory):
os.makedirs(email_directory)
# Display notice
print("PROCESSING: Email UID = {} from {}".format(uid, from_email))
# Add the subject
val_dict["Subject"] = self.__get_subject(email_message).strip()
# If the email has multiple parts
if email_message.is_multipart():
val_dict = self.__parse_multipart_message(
email_message, val_dict)
# If the message isn't multipart
else:
val_dict = self.__parse_singlepart_message(
email_message, val_dict)
msg_dict[key] = val_dict
# If required, move the email, mark it as unread, or delete it
self.__execute_options(uid, move, unread, delete)
# Return the dictionary of messages and their contents
return msg_dict
def __get_from(self, email_message):
"""Helper function for getting who an email message is from.
Args:
email_message (email.message): The email message to get sender of.
Returns:
A string containing the from email address.
"""
from_raw = email_message.get_all('From', [])
from_list = email.utils.getaddresses(from_raw)
if len(from_list[0]) == 1:
from_email = from_list[0][0]
elif len(from_list[0]) == 2:
from_email = from_list[0][1]
else:
from_email = "UnknownEmail"
return from_email
def __get_subject(self, email_message):
"""
"""
# Get the subject
subject = email_message.get("Subject")
# If there isn't a subject
if subject is None:
return "No Subject"
return subject
def __parse_multipart_message(self, email_message, val_dict):
"""Helper function for parsing multipart email messages.
Args:
email_message (email.message): The email message to parse.
val_dict (dict): A dictionary containing the message data from each
part of the message. Will be returned after it is updated.
Returns:
The dictionary containing the message data for each part of the
message.
"""
# For each part
for part in email_message.walk():
# If the part is an attachment
file_name = part.get_filename()
if bool(file_name):
# Generate file path
file_path = os.path.join(val_dict["email_directory"],
encoded_words_to_text(file_name))
file = open(file_path, 'wb')
file.write(part.get_payload(decode=True))
file.close()
# Get the list of attachments, or initialize it if there isn't one
attachment_list = val_dict.get("attachments") or []
attachment_list.append("{}".format(file_path))
val_dict["attachments"] = attachment_list
# If the part is html text
elif part.get_content_type() == 'text/html':
# Convert the body from html to plain text
val_dict["Plain_HTML"] = html2text.html2text(
part.get_payload())
val_dict["HTML"] = part.get_payload()
# If the part is plain text
elif part.get_content_type() == 'text/plain':
# Get the body
val_dict["Plain_Text"] = part.get_payload()
return val_dict
def __parse_singlepart_message(self, email_message, val_dict):
"""Helper function for parsing singlepart email messages.
Args:
email_message (email.message): The email message to parse.
val_dict (dict): A dictionary containing the message data from each
part of the message. Will be returned after it is updated.
Returns:
The dictionary containing the message data for each part of the
message.
"""
# Get the message body, which is plain text
val_dict["Plain_Text"] = email_message.get_payload()
return val_dict
def __execute_options(self, uid, move, unread, delete):
"""Loop through optional arguments and execute any required processing.
Args:
uid (int): The email ID to process.
move (str): The folder to move the emails to. If None, the emails
are not moved. Defaults to None.
unread (bool): Whether the emails should be marked as unread.
Defaults to False.
delete (bool): Whether the emails should be deleted. Defaults to
False.
Returns:
None
"""
# If the message should be marked as unread
if bool(unread):
self.server.remove_flags(uid, [SEEN])
# If a move folder is specified
if move is not None:
try:
# Move the message to another folder
self.server.move(uid, move)
except:
# Create the folder and move the message to the folder
self.server.create_folder(move)
self.server.move(uid, move)
# If the message should be deleted
elif bool(delete):
# Move the email to the trash
self.server.set_gmail_labels(uid, "\\Trash")
return
def listen(self, process_func=write_txt_file, **kwargs):
"""Listen in an email folder for incoming emails, and process them.
Args:
process_func (function): A function called to further process the
emails. The function must take only the list of file paths
returned by the scrape function as an argument. Defaults to the
example function write_txt_file in the email_processing module.
**kwargs (dict): Additional arguments for processing the email.
Optional arguments include:
move (str): The folder to move emails to. If not set, the
emails will not be moved.
unread (bool): Whether the emails should be marked as unread.
If not set, emails are kept as read.
delete (bool): Whether the emails should be deleted. If not
set, emails are not deleted.
Returns:
None
"""
# Ensure server is connected
if type(self.server) is not IMAPClient:
raise ValueError("server attribute must be type IMAPClient")
while True:
self.__idle(process_func=process_func, **kwargs)
return
def __idle(self, process_func=write_txt_file, **kwargs):
"""Helper function, idles in an email folder processing incoming emails.
Args:
process_func (function): A function called to further process the
emails. The function must take only the list of file paths
returned by the scrape function as an argument. Defaults to the
example function write_txt_file in the email_processing module.
**kwargs (dict): Additional arguments for processing the email.
Optional arguments include:
move (str): The folder to move emails to. If not set, the
emails will not be moved.
unread (bool): Whether the emails should be marked as unread.
If not set, emails are kept as read.
delete (bool): Whether the emails should be deleted. If not
set, emails are not deleted.
Returns:
None
"""
# Set the relevant kwarg variables
move = kwargs.get('move')
unread = bool(kwargs.get('unread'))
delete = bool(kwargs.get('delete'))
# Start idling
self.server.idle()
# Set idle timeout to 5 minutes
inner_timeout = get_time() + 60 * 5
# Until idle times out
while get_time() < inner_timeout:
# Check for a new response every 30 seconds
responses = self.server.idle_check(timeout=30)
# If there is a response
if responses:
print("Server sent:", responses)
# Suspend the idling
self.server.idle_done()
# Process the new emails
msgs = self.scrape(move=move, unread=unread, delete=delete)
# Run the process function
process_func(msgs)
# Restart idling
self.server.idle()
# Stop idling
self.server.idle_done()
return
class gmailPy(object):
def __init__(self):
self.IMAP_SERVER = 'imap.gmail.com'
self.ssl = True
self.myIMAPc = None
self.response = None
self.folders = []
def login(self, username, password):
self.myIMAPc = IMAPClient(self.IMAP_SERVER, ssl=self.ssl)
self.myIMAPc.login(username, password)
# Returns a list of all the folders for a particular account
def get_folders(self):
self.response = self.myIMAPc.list_folders()
for item in self.response:
self.folders.append(item[2].strip('u'))
return self.folders
# Returns the total number of messages in a folder
def get_mail_count(self, folder='Inbox'):
self.response = self.myIMAPc.select_folder(folder, True)
return self.response['EXISTS']
# Method to delete messages based on their size
def delete_bigmail(self, folder='Inbox'):
self.myIMAPc.select_folder(folder, False)
# Gets all the message ids of the messages which are not deleted in the folder
messages = self.myIMAPc.search(['NOT DELETED'])
print "%d messages that aren't deleted" % len(messages)
if len(messages) > 0:
print "You can exit by entering 0 or pressing CTRL+C \n"
else: print "There are no messages in the folder"
# Gets the message sizes for all the message ids returned in previous step
# Note: Just sends one request for all message ids with a return time < 10 ms
self.response = self.myIMAPc.fetch(messages, ['RFC822.SIZE'])
# Sorts the dictionary returned by fetch by size in descending order
sorted_response = sorted(self.response.iteritems(), key=operator.itemgetter(1), reverse=True)
count = 1
try:
for item in sorted_response:
# Gets the biggest message including headers, body, etc.
big_message = self.myIMAPc.fetch(item[0], ['RFC822'])
for msgid, data in big_message.iteritems():
msg_string = data['RFC822']
# Parses the message string using email library
msg = email.message_from_string(msg_string)
val = dict(self.response[msgid])['RFC822.SIZE']
print 'ID %d: From: %s Date: %s' % (msgid, msg['From'], msg['date'])
print 'To: %s' % (msg['To'])
print 'Subject: %s' % (msg['Subject'])
print 'Size: %d bytes \n' % (val)
user_del = raw_input("Do you want to delete this message?(Y/N): ")
if user_del == 'Y':
self.delete_message(msgid)
if count == len(sorted_response):
print "There are no more messages"
else:
print "\nMoving on to the next biggest message >>> \n"
elif user_del == '0':
print "Program exiting"
sys.exit()
else:
if count == len(sorted_response):
print "There are no more messages"
else:
print "\nMoving on to the next biggest message >>> \n"
count += 1
except KeyboardInterrupt:
print "Program exiting"
sys.exit()
# Method to delete messages based on their size with a search criteria
def delete_bigmail_search(self, folder='Inbox', command='', criteria=''):
self.myIMAPc.select_folder(folder, False)
# Gets all the message ids from the server based on the search criteria
messages = self.myIMAPc.search('%s "%s"' % (command, criteria))
print "%d messages that match --> %s: %s" % (len(messages), command, criteria)
if len(messages) > 0:
print "You can exit by entering 0 or pressing CTRL+C \n"
else: print "There are no messages in that matched your search criteria"
# Gets the message sizes for all the message ids returned in previous step
# Note: Just sends one request for all message ids with a return time < 10 ms
self.response = self.myIMAPc.fetch(messages, ['RFC822.SIZE'])
# Sorts the messages in decending order of their sizes
sorted_response = sorted(self.response.iteritems(), key=operator.itemgetter(1), reverse=True)
count = 1
try:
for item in sorted_response:
# Gets the entire content for the biggest message identified
big_message = self.myIMAPc.fetch(item[0], ['RFC822'])
for msgid, data in big_message.iteritems():
msg_string = data['RFC822']
msg = email.message_from_string(msg_string)
val = dict(self.response[msgid])['RFC822.SIZE']
print 'ID %d: From: %s Date: %s' % (msgid, msg['From'], msg['date'])
print 'To: %s' % (msg['To'])
print 'Subject: %s' % (msg['Subject'])
print 'Size: %d bytes \n' % (val)
user_del = raw_input("Do you want to delete this message?(Y/N): ")
if user_del == 'Y':
self.delete_message(msgid)
if count == len(sorted_response):
print "There are no more messages"
else:
print "\nMoving on to the next biggest message >>> \n"
elif user_del == '0':
print "Program exiting"
sys.exit()
else:
if count == len(sorted_response):
print "There are no more messages"
else:
print "\nMoving on to the next biggest message >>> \n"
count += 1
except KeyboardInterrupt:
print "Program exiting"
sys.exit()
# Deletes a message in the current folder based on msg id
def delete_message(self, id):
try:
self.myIMAPc.delete_messages([id])
self.myIMAPc.expunge()
print "Message deleted"
except IMAPClient.Error as err:
print "Message deletion failed"
print err
# Renames a folder
def rename_folder(self, oldfolder, newfolder):
try:
self.myIMAPc.rename_folder(oldfolder, newfolder)
print "Folder %s renamed to %s" % (oldfolder, newfolder)
except IMAPClient.Error as err:
print "Folder renaming failed"
print err
# Creates a new folder
def create_folder(self, folder):
try:
self.myIMAPc.create_folder(folder)
print "New folder %s created" % folder
except IMAPClient.Error as err:
print "Folder creation failed"
print err
# Deletes a folder
def delete_folder(self, folder):
try:
self.myIMAPc.delete_folder(folder)
print "Folder %s deleted" % folder
except IMAPClient.Error as err:
print "Folder deletion failed"
print err
# Creates a new folder and copies the content from the two folders that need to be merged
# Then deletes the old folders
def merge_folders(self, merged_folder, folder_1, folder_2):
try:
self.create_folder(merged_folder)
# Selects the folder with read/write permission
self.myIMAPc.select_folder(folder_1, True)
messages = self.myIMAPc.search(['NOT DELETED'])
print "Moving %d messages from %s to %s" % (len(messages), folder_1, merged_folder)
self.myIMAPc.copy(messages, merged_folder)
self.myIMAPc.select_folder(folder_2, True)
messages = self.myIMAPc.search(['NOT DELETED'])
print "Moving %d messages from %s to %s" % (len(messages), folder_2, merged_folder)
self.myIMAPc.copy(messages, merged_folder)
print "Deleting %s and %s..." % (folder_1, folder_2)
self.delete_folder(folder_1)
self.delete_folder(folder_2)
print "Merge folder operation succeeded"
except IMAPClient.Error as err:
print "Merge operation failed"
print err
def logout(self):
self.myIMAPc.logout()
