def create_folder_if_not_exists(client: IMAPClient, folder: str, logger: logging.Logger): """Create folder if it already exists.""" try: client.create_folder(folder) except IMAPClientError: # most likely, it already exists logger.info("Looks like the folder %s already exists.", folder)
def __init__(self, HOST, USERNAME, PASSWORD, ssl=False, folder='TODOLIST'): imap = IMAPClient(HOST, use_uid=True, ssl=ssl) imap.login(USERNAME, PASSWORD) try: imap.select_folder(folder) except imap.Error, e: imap.create_folder(folder)
def random_folder(logged_in_client: IMAPClient): folder = "random-122122121" if not logged_in_client.folder_exists(folder): logged_in_client.create_folder(folder) yield folder if logged_in_client.folder_exists(folder): logged_in_client.delete_folder(folder) assert not logged_in_client.folder_exists(folder)
def test001_imapclient_can_create_folder_in_imap(): """ Client can create folders in his mail. Steps: - Start imap server, should succeed. - List default folder, inbox should be there. - Create new folder, should succeed. """ global servers servers = [] cmd = "kosmos 'j.servers.imap.start()'" info("Execute {} in tmux main session".format(cmd)) pan = j.servers.tmux.execute(cmd=cmd) info("Wait for 30s to make sure that the server is running") sleep(30) info("Assert that the server is running") assert pan.cmd_running is True, "imap server is not running" servers.append(pan) info("List default folder, inbox should be there") box = Imbox("0.0.0.0", "*****@*****.**", "randomPW", ssl=False, port=7143) # assert "INBOX" in str(box.folders()[-1][0]) # assert the whole string instead of the first element in the tuple as it is ordered alphabetically. assert "INBOX" in str(box.folders()[-1]) info("Connect the client to the IMAP server") client = IMAPClient("0.0.0.0", port=7143, ssl=False) client.login("*****@*****.**", "randomPW") box_name = j.data.idgenerator.generateXCharID(10) info("Create {} box".format(box_name)) client.create_folder(box_name) info("Assert that the new box has been created") # assert box_name in str(box.folders()[-1][0]) # assert the whole string instead of the first element in the tuple as it is ordered alphabetically. assert box_name in str(box.folders()[-1])
def main(): s_username = raw_input("Source Email: ") s_password = getpass.getpass(prompt="Source Password: "******"Destination Email: ") d_password = getpass.getpass(prompt="Source Password: "******"Run it for real? (yes/*)") destination_folder = 'Migrated Chatlogs' # source server source = IMAPClient('imap.gmail.com', use_uid=True, ssl=True) source.login(s_username, s_password) # destination server destination = IMAPClient('imap.gmail.com', use_uid=True, ssl=True) destination.login(d_username, d_password) select_info = source.select_folder("[Gmail]/Chats", readonly=True) print 'Migrating %s chat messages' % select_info['EXISTS'] print 'Why don\'t you go grab a cup of coffee.. this is going to take a while' chats = source.search(['NOT DELETED']) if not destination.folder_exists(destination_folder): print "creating %s " % destination_folder destination.create_folder(destination_folder) for message in chats: print message fetchData = source.fetch(int(message), ['INTERNALDATE', 'FLAGS', 'RFC822'])[message] if certain == "yes": destination.append(destination_folder, msg=fetchData['RFC822'], flags=fetchData['FLAGS'], msg_time=fetchData['INTERNALDATE']) destination.logout() source.logout()
class MailGetter(object): """ Encapsulates behavior of retrieving email, saving PDFs, and moving emails. """ def __init__(self): """ Class initializer. """ mimetypes.init() logger = Logger() self.logger = logger.get_logger(MAIN + ".Cls") self.server = None self.queue = BotQueue() if not _check_paths(['input_path']): self.logger.fatal("File paths could not be created...cannot continue.") exit() def connect(self) -> bool: """ Connect and login to the remote IMAP server. Returns: (bool): True if successful, otherwise False """ mailserver = os.environ.get('mailserver') imapport = os.environ.get('imapport') mailssl = os.environ.get('mailssl') try: self.server = IMAPClient( mailserver, port=imapport, ssl=mailssl, use_uid=True ) username = os.environ.get('mail_username') password = os.environ.get('mail_password') self.logger.debug(f"Username: {username}, Password: {password}") self.server.login(username, password) except ConnectionRefusedError as e: self.logger.fatal(f"Connection to {mailserver}:{imapport} was refused: {str(e)}") return False except Exception as e: self.logger.fatal(f"Connection to {mailserver}:{imapport} was refused: {str(e)}") return False return True def reconnect(self) -> bool: """ Reconnect to the email server. Returns: (bool): True if successful, otherwise False """ self.logger.debug("Reconnecting to server.") try: self.server.idle_done() except Exception as e: self.logger.error("Error exiting IDLE mode: %s", e) self.logger.info("Will try to reconnect anyway.") try: self.server.logout() except Exception as e: self.logger.error("Error disconnecting from IMAP server: %s", e) self.logger.info("Will try to reconnect anyway.") return self.connect() def check_folders(self) -> bool: """ Check to see if we have the required INBOX and PROCESSED folders. If we do not have an inbox, we can't go on. If we do not have a processed folder, try to create it. Returns: (bool): True if successful, otherwise false. """ # First, make sure the INBOX folder exists. If it does not exist, # we have a serious problem and need to quit. inbox = os.environ.get('inbox') if not self.server.folder_exists(inbox): self.logger.fatal(f"Error locating INBOX named '{inbox}'.") return False # Next, see if the PROCESSED folder exists. If it does not, try to # create it. If we try to create it and the creation fails, again, # we have a serious problem and cannot continue. processed_folder = os.environ.get('processed_folder') if not self.server.folder_exists(processed_folder): self.logger.error(f"Error locating PROCESSED folder named '{processed_folder}'.") self.logger.error(f"Will attempt to create folder named '{processed_folder}'.") try: message = self.server.create_folder(processed_folder) self.logger.info( "Successfully created '%s': %s", processed_folder, message ) except Exception: self.logger.fatal( "Failed to create '%s': %s", processed_folder, message ) return False self.logger.info("Folder check was successful.") return True def save_linked_files( self, links: list, msgid: str, from_email: str, subject: str, reply_to: str ): """ Download and save files referenced by a link instead of directly attached to the email message. Raises exceptions rather than catches them so that the caller's error handler can deal with and record the error. Setting *allow_redirects* to ```True``` let's us retrieve files from cloud services such as DropBox and from URL smashers like www.tinyurl.com. Args: links (list): List of links to download from. msgid (str): ID of the email message we are processing. Used for filename disambiguation. from_email (str): Apparent sender of the email. subject (str): Subject line of the email. reply_to (str): Reply-To Address of the email """ input_path = os.environ.get('input_path') for link in links: if link[-4:].upper() == ".PDF": my_link = cloudize_link(link) self.logger.debug("Found link: %s", my_link) content = requests.get(my_link, allow_redirects=True).content filename = "{}/{}-{}".format( input_path, msgid, urllib.parse.unquote(link[link.rfind("/")+1:]) ) with open(filename, "wb") as fp: fp.write(content) self.queue.publish(arrival_notification( from_email, reply_to, subject, filename, 'application/pdf') ) def process_message(self, msgid, message) -> bool: """ Process one message. Args: msgid (str): Unique ID for this message. message (email): Email message to process. Returns: (bool): True if successful, otherwise False """ self.logger.debug( "ID #%s: From: %s; Subject: %s", msgid, message.get("From"), message.get("Subject") ) from_email = sanitize_from_name(message.get("From")) reply_to = sanitize_from_name(message.get("Return-Path") or from_email) input_path = os.environ.get('input_path') try: subject = message.get("Subject") except OSError as e: self.logger.error("Cannot extract message subject: %s", str(e)) subject = "N/A" for key in message.keys(): self.logger.debug("%s = %s", key, message.get(key, None)) # Go through each part of the message. If we find a pdf file, save it. counter = 1 # number of attachments we've processed for this message. for part in message.walk(): try: # multipart/* are just containers...skip them if part.get_content_maintype() == "multipart": continue # Extract & sanitize filename. Create a filename if not given. filename = part.get_filename() self.logger.debug("*** %s ***", filename) extension = mimetypes.guess_extension(part.get_content_type())\ or ".bin" if not filename: filename = "{}-{}-part-{}{}".format( msgid, from_email, counter, extension ) else: filename = "{}-{}"\ .format(msgid, filename)\ .replace("\r", "")\ .replace("\n", "") counter += 1 # Save the attached file # For now, only save files of type ".PDF" # TODO: Parse HTML parts to see if we have links to PDF files # stored elsewhere. lower_extension = extension.lower() # Save attached file . . . if lower_extension in ['.pdf', '.docx', '.doc', '.rtf']: try: mimetype = mimetypes.types_map[lower_extension] except KeyError as error: self.logger.error( "Cannot map '%s' to a mime type: %s", lower_extension, str(error) ) mimetype = None if mimetype is not None: filename = "{}/{}".format( input_path, filename ) with open(filename, "wb") as fp: fp.write(part.get_payload(decode=True)) self.queue.publish(arrival_notification( from_email, reply_to, subject, filename, mimetype) ) # Save file referenced by a link . . . elif lower_extension[:4] == ".htm": links = extract_html_links(part.get_payload()) self.save_linked_files( links, msgid, from_email, subject, reply_to ) elif lower_extension == ".bat": links = extract_text_links(part.get_payload()) self.save_linked_files( links, msgid, from_email, subject, reply_to ) else: self.logger.info("Skipping: %s", filename) except Exception as e: self.logger.error( "Error with attachment #%s from message #%s from %s: %s", counter, msgid, message.get("From"), e ) self.logger.exception(e) return False return True def wait_for_messages(self): # FIRST: Process anything that's already in our INBOX self.process_inbox() # NEXT: Go into IDLE mode waiting for either a timeout or new mail. self.server.idle() stay_alive = True # Number of times we've returned from idle without receiving any # messages. idle_counter = 0 # Number of seconds to wait for more messages before timing out. idle_timeout = 60 # If we don't receive something within this many seconds, we'll # reconnect to the server. reconnect_seconds = 300 while stay_alive: try: # Wait for up to *idle_timeout* seconds for new messages. responses = self.server.idle_check(timeout=idle_timeout) self.logger.debug("Response to idle_check(): %s", responses) if responses: # We DID get new messages. Process them. idle_counter = 0 responses = self.server.idle_done() self.logger.debug("Response to idle_done(): %s", responses) self.process_inbox() self.server.idle() else: # We did not get any new messages. idle_counter += 1 # If we've run out of patience, reconnect and resume idle # mode. if idle_counter * idle_timeout > reconnect_seconds: if self.reconnect(): idle_counter = 0 self.process_inbox() self.server.idle() # Reconnect failure (!) else: stay_alive = False except imaplib.IMAP4.abort as e: self.logger.error("IMAP connection closed by host: %s", e) self.logger.error("Will try to reconnect") if self.reconnect(): idle_counter = 0 self.process_inbox() self.server.idle() else: self.logger.error("Unable to reconnect. Shutting down.") stay_alive = False except Exception as e: self.logger.error("Error in IDLE loop: %s", e) self.logger.exception(e) self.logger.error("Shutting down due to the above errors.") stay_alive = False def process_inbox(self): """ Process each message in the INBOX. After a message is processed: If processing was successful: Mark as "SEEN" so that we don't process it again. If processing was not successful: Mark as "SEEN" and "FOLLOWUP" so that an operator can fix if and requeue it by clearing the SEEN flag. """ select_info = self.server.select_folder(os.environ.get('inbox')) self.logger.debug( "%d messages in %s.", select_info[b'EXISTS'], os.environ.get('inbox') ) messages = self.server.search(criteria='UNSEEN') for msgid, data in self.server.fetch(messages, ['RFC822']).items(): email_message = email.message_from_bytes(data[b'RFC822']) if self.process_message(msgid, email_message): # Mark message as "Seen". For now, we *won't* move the message # to the PROCESSED folder. self.server.set_flags(msgid, b'\\Seen') else: # If we had an error processing the message, mark it for # follow AND as seen. self.server.set_flags( msgid, [b'\\Flagged for Followup', b'\\Seen'] )
class SourceDriverMail(SourceDriver): def __init__(self, instanceName: str, settings: Settings, parser: MessageParser) -> None: super().__init__("mail", instanceName, settings, parser) # Settings self.__server = self.getSettingString("server", "") self.__user = self.getSettingString("user", "") self.__password = self.getSettingString("password", "") self.__ssl = self.getSettingBoolean("ssl", True) self.__fix_weak_dh = self.getSettingBoolean("fix_weak_dh", False) self.__allowlist = self.getSettingList("allowlist", []) self.__denylist = self.getSettingList("denylist", []) self.__cleanup = self.getSettingString("cleanup", "") self.__archive_folder = self.getSettingString("archive_folder", "Archive") if self.__cleanup == "Delete": self.print("Cleanup strategy is delete") elif self.__cleanup == "Archive": self.print("Cleanup strategy is archive") elif self.__cleanup == "": self.print("Cleanup is disabled") else: self.fatal("Unknown cleanup strategy") # Internal self.__healthy = False self.__connect() def retrieveEvent(self) -> Optional[SourceEvent]: try: if self.isDebug(): self.print("Checking for new mails") messages = self.__imap_client.search('UNSEEN') for uid, message_data in self.__imap_client.fetch( messages, "RFC822").items(): message = email.message_from_bytes(message_data[b"RFC822"], policy=policy.default) sender = parseaddr(message.get("From"))[1] sourceEvent = SourceEvent() sourceEvent.source = SourceEvent.SOURCE_MAIL sourceEvent.timestamp = datetime.datetime.strptime( message.get('Date'), "%a, %d %b %Y %H:%M:%S %z").strftime( SourceEvent.TIMESTAMP_FORMAT) sourceEvent.sender = sender sourceEvent.raw = message.get_body(('plain', )).get_content() if self.isSenderAllowed(allowlist=self.__allowlist, denylist=self.__denylist, sender=sender): parsedSourceEvent = self.parser.parseMessage( sourceEvent, None) # type: ignore[union-attr] self.__do_cleanup(uid) return parsedSourceEvent else: self.error("Received unhandled message (ignored sender)") return UnhandledEvent.fromSourceEvent( sourceEvent, UnhandledEvent.CAUSE_IGNORED_SENDER) except (timeout, OSError) as e: self.error("Connection to mailserver timed out") self.__healthy = False self.__connect() def getSourceState(self) -> SourceState: if self.__healthy: return SourceState.OK return SourceState.ERROR def __connect(self): if self.__fix_weak_dh: context = ssl.SSLContext( ssl.PROTOCOL_TLSv1_2) # Workaround for weak dh key context.set_ciphers('DEFAULT@SECLEVEL=1') else: context = ssl.SSLContext() try: if self.isDebug(): self.print("Connecting to server {}".format(self.__server)) self.__imap_client = IMAPClient(self.__server, use_uid=True, ssl=self.__ssl, ssl_context=context, timeout=1.0) except gaierror: self.error("Failed to connect to Mail Server") except ssl.SSLError: self.fatal("Failed to connect to Mail Server (TLS Error)") else: try: if self.isDebug(): self.print("Login as user {}".format(self.__user)) self.__imap_client.login(self.__user, self.__password) except LoginError: self.error("Mail Server login failed") else: if self.isDebug(): self.print("Login successful") self.__healthy = True self.__imap_client.select_folder('INBOX', readonly=False) def __create_imap_folder(self, folder): if not self.__imap_client.folder_exists(folder): self.print("Folder {} does not exist creating") self.__imap_client.create_folder(folder) def __do_cleanup(self, uid): if self.__cleanup == "Archive": self.__create_imap_folder(self.__archive_folder) self.__imap_client.copy(uid, self.__archive_folder) if self.__cleanup == "Delete" or self.__cleanup == "Archive": self.__imap_client.delete_messages(uid) self.__imap_client.expunge(uid)
class ImapConn(object): def __init__(self, db, foldername, conn_info): self.db = db self.foldername = foldername self._thread = None self.MHOST, self.MUSER, self.MPASSWORD = conn_info self.event_initial_polling_complete = threading.Event() self.pending_imap_jobs = False # persistent database state below self.db_folder = self.db.setdefault(foldername, {}) self.db_messages = self.db.setdefault(":message-full", {}) last_sync_uid = db_folder_attr("last_sync_uid") @contextlib.contextmanager def wlog(self, msg): t = time.time() - started with lock_log: print("%03.2f [%s] %s -->" % (t, self.foldername, msg)) t0 = time.time() yield t1 = time.time() with lock_log: print("%03.2f [%s] ... finish %s (%3.2f secs)" % (t1 - started, self.foldername, msg, t1 - t0)) def log(self, *msgs): t = time.time() - started bmsg = "%03.2f [%s]" % (t, self.foldername) with lock_log: print(bmsg, *msgs) def connect(self): with self.wlog("IMAP_CONNECT {}: {}".format(self.MUSER, self.MPASSWORD)): ssl_context = ssl.create_default_context() # don't check if certificate hostname doesn't match target hostname ssl_context.check_hostname = False # don't check if the certificate is trusted by a certificate authority ssl_context.verify_mode = ssl.CERT_NONE self.conn = IMAPClient(self.MHOST, ssl_context=ssl_context) self.conn.login(self.MUSER, self.MPASSWORD) self.log(self.conn.welcome) try: self.select_info = self.conn.select_folder(self.foldername) except IMAPClientError: self.ensure_folder_exists() self.select_info = self.conn.select_folder(self.foldername) self.log('folder has %d messages' % self.select_info[b'EXISTS']) self.log('capabilities', self.conn.capabilities()) def ensure_folder_exists(self): with self.wlog("ensure_folder_exists: {}".format(self.foldername)): try: resp = self.conn.create_folder(self.foldername) except IMAPClientError as e: if "ALREADYEXISTS" in str(e): return print("EXCEPTION:" + str(e)) else: print("Server sent:", resp if resp else "nothing") def move(self, messages): self.log("IMAP_MOVE to {}: {}".format(MVBOX, messages)) try: resp = self.conn.move(messages, MVBOX) except IMAPClientError as e: if "EXPUNGEISSUED" in str(e): self.log( "IMAP_MOVE errored with EXPUNGEISSUED, probably another client moved it" ) else: self.log( "IMAP_MOVE {} successfully completed.".format(messages)) def perform_imap_idle(self): if self.pending_imap_jobs: self.log("perform_imap_idle skipped because jobs are pending") return with self.wlog("IMAP_IDLE()"): res = self.conn.idle() interrupted = False while not interrupted: # Wait for up to 30 seconds for an IDLE response responses = self.conn.idle_check(timeout=30) self.log("Server sent:", responses if responses else "nothing") for resp in responses: if resp[1] == b"EXISTS": # we ignore what is returned and just let # perform_imap_fetch look since lastseen # id = resp[0] interrupted = True resp = self.conn.idle_done() def perform_imap_fetch(self): range = "%s:*" % (self.last_sync_uid + 1, ) with self.wlog("IMAP_PERFORM_FETCH %s" % (range, )): requested_fields = [ b"RFC822.SIZE", b'FLAGS', b"BODY.PEEK[HEADER.FIELDS (FROM TO CC DATE CHAT-VERSION MESSAGE-ID IN-REPLY-TO)]" ] resp = self.conn.fetch(range, requested_fields) timestamp_fetch = time.time() for uid in sorted(resp): # get lower uids first if uid < self.last_sync_uid: self.log( "IMAP-ODDITY: ignoring bogus uid %s, it is lower than min-requested %s" % (uid, self.last_sync_uid)) continue data = resp[uid] headers = data[requested_fields[-1].replace(b'.PEEK', b'')] msg_headers = email.message_from_bytes(headers) message_id = normalized_messageid(msg_headers) chat_version = msg_headers.get("Chat-Version") in_reply_to = msg_headers.get("In-Reply-To", "").lower() if not self.has_message(normalized_messageid(msg_headers)): self.log('fetching body of ID %d: %d bytes, message-id=%s ' 'in-reply-to=%s chat-version=%s' % ( uid, data[b'RFC822.SIZE'], message_id, in_reply_to, chat_version, )) fetchbody_resp = self.conn.fetch(uid, [b'BODY.PEEK[]']) msg = email.message_from_bytes( fetchbody_resp[uid][b'BODY[]']) msg.fetch_retrieve_time = timestamp_fetch msg.foldername = self.foldername msg.uid = uid msg.move_state = DC_CONSTANT_MSG_MOVESTATE_PENDING self.store_message(message_id, msg) else: msg = self.get_message_from_db(message_id) self.log('fetching-from-db: ID %s message-id=%s' % (uid, message_id)) if msg.foldername != self.foldername: self.log("detected moved message", message_id) msg.foldername = self.foldername msg.move_state = DC_CONSTANT_MSG_MOVESTATE_STAY if self.foldername in (INBOX, SENT): if self.resolve_move_status( msg) != DC_CONSTANT_MSG_MOVESTATE_PENDING: # see if there are pending messages which have a in-reply-to # to our currnet msg # NOTE: should be one sql-statement to find the # possibly multiple messages that waited on us for dbmid, dbmsg in self.db_messages.items(): if dbmsg.move_state == DC_CONSTANT_MSG_MOVESTATE_PENDING: if dbmsg.get("In-Reply-To", "").lower() == message_id: self.log("resolving pending message", dbmid) # resolving the dependent message must work now res = self.resolve_move_status(dbmsg) assert res != DC_CONSTANT_MSG_MOVESTATE_PENDING, ( dbmid, res) if not self.has_message(message_id): self.store_message(message_id, msg) self.last_sync_uid = max(uid, self.last_sync_uid) self.log("last-sync-uid after fetch:", self.last_sync_uid) self.db.sync() def resolve_move_status(self, msg): """ Return move-state after this message's next move-state is determined (i.e. it is not PENDING)""" message_id = normalized_messageid(msg) if msg.move_state == DC_CONSTANT_MSG_MOVESTATE_PENDING: res = self.determine_next_move_state(msg) if res == DC_CONSTANT_MSG_MOVESTATE_MOVING: self.schedule_move(msg) msg.move_state = DC_CONSTANT_MSG_MOVESTATE_MOVING elif res == DC_CONSTANT_MSG_MOVESTATE_STAY: self.log("STAY uid=%s message-id=%s" % (msg.uid, message_id)) msg.move_state = DC_CONSTANT_MSG_MOVESTATE_STAY else: self.log("PENDING uid=%s message-id=%s in-reply-to=%s" % (msg.uid, message_id, msg["In-Reply-To"])) return msg.move_state def determine_next_move_state(self, msg): """ Return the next move state for this message. Only call this function if the message is pending. This function works with the DB, does not perform any IMAP commands. """ self.log("shall_move %s " % (normalized_messageid(msg))) assert self.foldername in (INBOX, SENT) assert msg.move_state == DC_CONSTANT_MSG_MOVESTATE_PENDING if msg.foldername == MVBOX: self.log("is already in mvbox, next state is STAY %s" % (normalized_messageid(msg))) return DC_CONSTANT_MSG_MOVESTATE_STAY last_dc_count = 0 while 1: last_dc_count = (last_dc_count + 1) if is_dc_message(msg) else 0 in_reply_to = normalized_messageid(msg.get("In-Reply-To", "")) if not in_reply_to: type_msg = "DC" if last_dc_count else "CLEAR" self.log("detected thread-start %s message" % type_msg, normalized_messageid(msg)) if last_dc_count > 0: return DC_CONSTANT_MSG_MOVESTATE_MOVING else: return DC_CONSTANT_MSG_MOVESTATE_STAY newmsg = self.get_message_from_db(in_reply_to) if not newmsg: self.log("failed to fetch from db:", in_reply_to) # we don't have the parent message ... maybe because # it hasn't arrived (yet), was deleted or we failed to # scan/fetch it: if last_dc_count >= 4: self.log( "no thread-start found, but last 4 messages were DC") return DC_CONSTANT_MSG_MOVESTATE_MOVING else: self.log("pending: missing parent, last_dc_count=%x" % (last_dc_count, )) return DC_CONSTANT_MSG_MOVESTATE_PENDING elif newmsg.move_state == DC_CONSTANT_MSG_MOVESTATE_MOVING: self.log("parent was a moved message") return DC_CONSTANT_MSG_MOVESTATE_MOVING else: msg = newmsg assert 0, "should never arrive here" def schedule_move(self, msg): message_id = normalized_messageid(msg) assert msg.foldername != MVBOX self.log("scheduling move message-id=%s" % (message_id)) self.pending_imap_jobs = True def has_message(self, message_id): assert isinstance(message_id, str) return message_id in self.db_messages def get_message_from_db(self, message_id): return self.db_messages.get(normalized_messageid(message_id)) def store_message(self, message_id, msg): mid2 = normalized_messageid(msg) message_id = normalized_messageid(message_id) assert message_id == mid2 assert message_id not in self.db_messages, message_id assert msg.foldername in (MVBOX, SENT, INBOX) self.db_messages[message_id] = msg self.log("stored new message message-id=%s" % (message_id, )) def forget_about_too_old_pending_messages(self): # some housekeeping but not sure if neccessary # because the involved sql-statements # probably don't care if there are some foreever-pending messages now = time.time() for dbmid, dbmsg in self.db_messages.items(): if dbmsg.move_state == DC_CONSTANT_MSG_MOVESTATE_PENDING: delay = now - dbmsg.fetch_retrieve_time if delay > self.pendingtimeout: dbmsg.move_state = DC_CONSTANT_MSG_MOVESTATE_STAY self.log("pendingtimeout: message now set to stay", dbmid) def perform_imap_jobs(self): with self.wlog("perform_imap_jobs()"): if self.foldername in (INBOX, SENT): to_move_uids = [] to_move_msgs = [] # determine all uids of messages that are to be moved for dbmid, dbmsg in self.db_messages.items(): if dbmsg.move_state == DC_CONSTANT_MSG_MOVESTATE_MOVING: if dbmsg.uid > 0: # else it's already moved? to_move_uids.append(dbmsg.uid) to_move_msgs.append(dbmsg) if to_move_uids: self.move(to_move_uids) # now that we moved let's invalidate "uid" because it's # not there anyore in thie folder for dbmsg in to_move_msgs: dbmsg.uid = 0 self.pending_imap_jobs = False def _run_in_thread(self): self.connect() if self.foldername == INBOX: # INBOX loop should wait until MVBOX polled once mvbox.event_initial_polling_complete.wait() now = time.time() while True: self.perform_imap_jobs() self.perform_imap_fetch() if self.foldername == MVBOX: # signal that MVBOX has polled once self.event_initial_polling_complete.set() elif self.foldername == INBOX: # it's not clear we need to do this housekeeping # (depends on the SQL statements) self.forget_about_too_old_pending_messages() self.perform_imap_idle() def start_thread_loop(self): assert not self._thread self._thread = t = threading.Thread(target=self._run_in_thread) t.start()
by_size = sorted(list(response.viewitems()), key=lambda m: m[1]['RFC822.SIZE']) sizes = [ m[1]['RFC822.SIZE'] for m in by_size ] # select msgs that are bigger than threshold bigger = bisect.bisect_right(sizes, options.threshold) # index of the first message bigger than threshold big_messages = by_size[bigger:] print "There are %d messages bigger than %s" % (len(big_messages), options.threshold) big_uids = [ msg[0] for msg in big_messages ] def print_messages(): print "\n--- Messages bigger than {}: ".format(options.threshold).ljust(80, '-') print "Format: <date> | <size> | <from> | <subject>" for msgid, data in big_messages: headers = email.message_from_string(data['RFC822.HEADER']) size = data['RFC822.SIZE'] date = data['INTERNALDATE'] print "{} | {} | {} | {}".format(date, size, headers['from'], headers['subject']) if not options.no_label: print "Labeling big messages with label %s" % options.label server.create_folder(options.label) server.copy(big_uids, options.label) print "Your messages larger than {} bytes have been labeled with label {}".format(options.threshold, options.label) if options.print_msgs or options.no_label: print_messages()
class DirectImap: def __init__(self, account: Account) -> None: self.account = account self.logid = account.get_config("displayname") or id(account) self._idling = False self.connect() def connect(self): host = self.account.get_config("configured_mail_server") port = int(self.account.get_config("configured_mail_port")) security = int(self.account.get_config("configured_mail_security")) user = self.account.get_config("addr") pw = self.account.get_config("mail_pw") if security == const.DC_SOCKET_PLAIN: ssl_context = None else: ssl_context = ssl.create_default_context() # don't check if certificate hostname doesn't match target hostname ssl_context.check_hostname = False # don't check if the certificate is trusted by a certificate authority ssl_context.verify_mode = ssl.CERT_NONE if security == const.DC_SOCKET_STARTTLS: self.conn = IMAPClient(host, port, ssl=False) self.conn.starttls(ssl_context) elif security == const.DC_SOCKET_PLAIN: self.conn = IMAPClient(host, port, ssl=False) elif security == const.DC_SOCKET_SSL: self.conn = IMAPClient(host, port, ssl_context=ssl_context) self.conn.login(user, pw) self.select_folder("INBOX") def shutdown(self): try: self.conn.idle_done() except (OSError, IMAPClientError): pass try: self.conn.logout() except (OSError, IMAPClientError): print("Could not logout direct_imap conn") def create_folder(self, foldername): try: self.conn.create_folder(foldername) except imaplib.IMAP4.error as e: print("Can't create", foldername, "probably it already exists:", str(e)) def select_folder(self, foldername): assert not self._idling return self.conn.select_folder(foldername) def select_config_folder(self, config_name): """ Return info about selected folder if it is configured, otherwise None. """ if "_" not in config_name: config_name = "configured_{}_folder".format(config_name) foldername = self.account.get_config(config_name) if foldername: return self.select_folder(foldername) def list_folders(self): """ return list of all existing folder names""" assert not self._idling folders = [] for meta, sep, foldername in self.conn.list_folders(): folders.append(foldername) return folders def delete(self, range, expunge=True): """ delete a range of messages (imap-syntax). If expunge is true, perform the expunge-operation to make sure the messages are really gone and not just flagged as deleted. """ self.conn.set_flags(range, [DELETED]) if expunge: self.conn.expunge() def get_all_messages(self): assert not self._idling # Flush unsolicited responses. IMAPClient has problems # dealing with them: https://github.com/mjs/imapclient/issues/334 # When this NOOP was introduced, next FETCH returned empty # result instead of a single message, even though IMAP server # can only return more untagged responses than required, not # less. self.conn.noop() return self.conn.fetch(ALL, [FLAGS]) def get_unread_messages(self): assert not self._idling res = self.conn.fetch(ALL, [FLAGS]) return [uid for uid in res if SEEN not in res[uid][FLAGS]] def mark_all_read(self): messages = self.get_unread_messages() if messages: res = self.conn.set_flags(messages, [SEEN]) print("marked seen:", messages, res) def get_unread_cnt(self): return len(self.get_unread_messages()) def dump_imap_structures(self, dir, logfile): assert not self._idling stream = io.StringIO() def log(*args, **kwargs): kwargs["file"] = stream print(*args, **kwargs) empty_folders = [] for imapfolder in self.list_folders(): self.select_folder(imapfolder) messages = list(self.get_all_messages()) if not messages: empty_folders.append(imapfolder) continue log("---------", imapfolder, len(messages), "messages ---------") # get message content without auto-marking it as seen # fetching 'RFC822' would mark it as seen. requested = [b'BODY.PEEK[]', FLAGS] for uid, data in self.conn.fetch(messages, requested).items(): body_bytes = data[b'BODY[]'] if not body_bytes: log("Message", uid, "has empty body") continue flags = data[FLAGS] path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder) path.mkdir(parents=True, exist_ok=True) fn = path.joinpath(str(uid)) fn.write_bytes(body_bytes) log("Message", uid, fn) email_message = email.message_from_bytes(body_bytes) log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id")) if empty_folders: log("--------- EMPTY FOLDERS:", empty_folders) print(stream.getvalue(), file=logfile) def idle_start(self): """ switch this connection to idle mode. non-blocking. """ assert not self._idling res = self.conn.idle() self._idling = True return res def idle_check(self, terminate=False): """ (blocking) wait for next idle message from server. """ assert self._idling self.account.log("imap-direct: calling idle_check") res = self.conn.idle_check(timeout=30) if len(res) == 0: raise TimeoutError if terminate: self.idle_done() self.account.log("imap-direct: idle_check returned {!r}".format(res)) return res def idle_wait_for_seen(self): """ Return first message with SEEN flag from a running idle-stream REtiurn. """ while 1: for item in self.idle_check(): if item[1] == FETCH: if item[2][0] == FLAGS: if SEEN in item[2][1]: return item[0] def idle_done(self): """ send idle-done to server if we are currently in idle mode. """ if self._idling: res = self.conn.idle_done() self._idling = False return res def append(self, folder, msg): """Upload a message to *folder*. Trailing whitespace or a linebreak at the beginning will be removed automatically. """ if msg.startswith("\n"): msg = msg[1:] msg = '\n'.join([s.lstrip() for s in msg.splitlines()]) self.conn.append(folder, msg) def get_uid_by_message_id(self, message_id): msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id]) if len(msgs) == 0: raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?") return msgs[0]
class imap: def __init__(self): try: dbfile = open('.credentials', 'rb') db = pickle.load(dbfile) temp_server = db["server"] if (temp_server == ""): self.server_fail_flag = 1 # declaring that server object of imap is not initialised pass else: try: self.server = IMAPClient(temp_server, use_uid=True) self.server_fail_flag = 0 except: self.server_fail_flag = 1 # declaring that server object of imap is not initialised print( "Couldn't able to sign in. Please try again with valied credentials" ) except: self.server_fail_flag = 1 # declaring that server object of imap is not initialised file = open(".credentials", "w+") file.close() def login(self, id, passwd, server): self.server_name = server self.email = id self.passwd = passwd try: if (self.server_fail_flag == 1): self.server = IMAPClient(self.server_name, use_uid=True) self.server.login(self.email, self.passwd) return 1 print("Login successful") except: return 0 def logout(self): print("Logged out") self.server.logout() def list_folders(self): print("LIST OF FOLDERS") folders = self.server.list_folders() for i in folders: print(i[-1]) return folders print("") print(server.get_gmail_labels(messages)) def createFolder(self, folderName): self.server.create_folder(folderName) print("Folder named - %s - is created successfully..." % folderName) def deleteFolder(self, folderName): self.server.delete_folder(folderName) def deleteEmail(self, messages): self.server.delete_messages(messages) def list_mails(self, folder_name): select_info = self.server.select_folder(folder_name) print('%d messages in %s' % (select_info[b'EXISTS'], folder_name)) # messages = server.search(['FROM', '*****@*****.**']) messages = self.server.search(['ALL']) # messages = self.server.sort(['ARRIVAL']) print("%s messages from %s" % (len(messages), folder_name)) fetched_msgs = self.server.fetch(messages, ['ENVELOPE']).items() return fetched_msgs def search_mails(self, catagory, parameter): if (catagory in ["FROM", "TEXT", "SINCE", "SUBJECT"]): print([catagory, parameter]) messages = self.server.search([catagory, parameter]) print("%s messages from Search catagory -> %s" % (len(messages), catagory)) fetched_msgs = self.server.fetch(messages, ['ENVELOPE']).items() print("fetched Messages are ", fetched_msgs) return fetched_msgs else: print([catagory]) messages = self.server.search([catagory]) print(messages) print("%s messages from Search catagory -> %s" % (len(messages), catagory)) fetched_msgs = self.server.fetch(messages, ['ENVELOPE']).items() print("fetched Messages are ", fetched_msgs) return fetched_msgs def fetch_email_content(self, msg_id): for msgid, data in self.server.fetch(msg_id, 'RFC822').items(): # print("\nMESSAGE", msgid) # print("\nDATA IS => \n", data) # envelope = data[b'ENVELOPE'] msg = email.message_from_bytes(data[b'RFC822']) # print(msgid, email_message.get('From'), email_message.get('Subject')) # print(email_message.get('To'), email_message.get('body')) subject = msg.get('Subject') if msg.is_multipart(): # iterate over email parts for part in msg.walk(): # extract content type of email content_type = part.get_content_type() content_disposition = str(part.get("Content-Disposition")) try: # get the email body body = part.get_payload(decode=True).decode() except: pass if content_type == "text/plain" and "attachment" not in content_disposition: # print text/plain emails and skip attachments print(body) elif "attachment" in content_disposition: # download attachment filename = part.get_filename() if filename: if not os.path.isdir(subject): # make a folder for this email (named after the subject) os.mkdir(subject) filepath = os.path.join(subject, filename) # download attachment and save it open(filepath, "wb").write(part.get_payload(decode=True)) else: # extract content type of email content_type = msg.get_content_type() # get the email body body = msg.get_payload(decode=True).decode() if content_type == "text/plain": # print only text email parts print(body) if content_type == "text/html": # if it's HTML, create a new HTML file and open it in browser if not os.path.isdir(subject): # make a folder for this email (named after the subject) os.mkdir(subject) filename = "{}.html".format(subject[:50]) filepath = os.path.join(subject, filename) # write the file open(filepath, "w").write(body) # some data about mail to = msg.get("To") mail_from = msg.get("From") mail_date = msg.get("Date") mail_subject = msg.get("Subject") print("To =>", to) print("From =>", mail_from) print("Date =>", mail_date) print("=" * 100) # returning the useful values return (filepath, body, to, mail_from, mail_date, mail_subject)
class IMAP(Thread): def __init__(self, filman): super(IMAP, self).__init__() self.imap = IMAPClient(HOST, use_uid=True, ssl=ssl) self.imap.login(USERNAME, PASSWORD) self.messages = [] self.filterman = filman self.counter = 0 self.check_dests() self.loop() def check_dests(self): dests = self.filterman.get_dests() for d in dests: if not self.imap.folder_exists(d): self.imap.create_folder(d) logging.info('[create folder] %s' % d) if not self.imap.folder_exists(default_not_matched_dest): self.imap.create_folder(default_not_matched_dest) logging.info('[create folder] %s' % default_not_matched_dest) def mark_as_unread(self, msgs): return self.imap.remove_flags(msgs, ('\\SEEN')) def check(self): server = self.imap select_info = server.select_folder('INBOX') logging.info("source imap inited: %r" % select_info) messages = server.search(['NOT SEEN']) messages = sorted(messages, reverse=True) self.messages = list(messages) logging.info('got %d unread messages' % len(self.messages)) def idle(self, secs=30): server = self.imap server.idle() responses = server.idle_check(timeout=secs) text, responses = server.idle_done() logging.info('idle response: %s' % (responses)) return not responses def loop(self): logging.info('enter loop %d' % self.counter) self.counter += 1 self.check() while self.messages: self._dozen() self.imap.close_folder() def _dozen(self): if self.messages: msgs = self.messages[:12] self.messages = self.messages[12:] else: return logging.info('processing the first %d msgs; left %d...' % ( len(msgs), len(self.messages))) logging.info(msgs) response = self.imap.fetch(msgs, ['RFC822']) msgs = [(msgid, Msg(string=data['RFC822'])) for (msgid, data) in response.iteritems()] self.filterman.test_match_and_take_action(self.imap, msgs) def run(self): count = 0 while True: count += 1 logging.info('idle counter: %d' % count) self.idle() or self.loop() sleep(10) if not count % 5: # do loop every 10 runs. self.loop()
class IMAPBot(object): IMAPBotError = IMAPBotError def __init__(self, host, username, password, ssl=True): self.server = IMAPClient(host, use_uid=True, ssl=ssl) self.server.login(username, password) if 'IDLE' not in self.server.capabilities(): raise IMAPBotError('Sorry, this IMAP server does not support IDLE.') # the folder where processed emails go self.processed_folder = 'imapbot_processed' self.idle_timeout = 5 # seconds self._is_idle = False self._run = True self._create_folder(self.processed_folder) def check_mail(self): select_info = self.server.select_folder('INBOX') print '%d messages in INBOX' % select_info['EXISTS'] messages = self.server.search(['UNSEEN']) messages = self.server.search(['NOT DELETED']) print "%d messages that haven't been seen" % len(messages) if not messages: return #response = self.server.fetch(messages, ['FLAGS', 'INTERNALDATE', 'RFC822.SIZE', 'ENVELOPE', 'RFC822.TEXT']) response = self.server.fetch(messages, ['FLAGS', 'ENVELOPE', 'RFC822.TEXT']) for message_id, data in response.iteritems(): message = Message(message_id, data['ENVELOPE'], data['RFC822.TEXT'], data['FLAGS']) self.process(message) def complete(self, message): message_ids = [message.id] self.server.copy(message_ids, self.processed_folder) self.server.delete_messages(message_ids) self.server.expunge() def _create_folder(self, name): # make sure the folder doesn't already exist if self.server.folder_exists(name): return self.server.create_folder(name) def handle_message(self, message): print 'message id: {}, from: {}:'.format(message.id, message.envelope.get_email('from')) with open('message.txt', 'ab') as fh: fh.write('{}\n\n'.format(message.text)) print message.plain or message.html or 'no message' def idle(self): if self._is_idle: return self.server.idle() self._is_idle = True return True # this actually changed state def unidle(self): if not self._is_idle: return self.server.idle_done() self._is_idle = False return True # this call actually changed state def process(self, message): self.handle_message(message) self.complete(message) def run(self): # process any mail that was in the inbox before coming online self.check_mail() # put the connection in idle mode so we get notifications self.idle() # loop forever looking for stuff while self._run: for message in self.server.idle_check(timeout=self.idle_timeout): if message[0] == 'OK': continue with Unidle(self): self.check_mail() def quit(self): self._run = False self.unidle() print self.server.logout()
class IMAP(): """ Central class for IMAP server communication """ Retval = namedtuple('Retval', 'code data') def __init__(self, logger, username, password, server='localhost', port=143, starttls=False, imaps=False, tlsverify=True, test=False, timeout=None): self.logger = logger self.username = username self.password = password self.server = server self.port = port self.imaps = imaps self.starttls = starttls self.timeout = timeout self.sslcontext = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) if tlsverify: self.sslcontext.verify_mode = ssl.CERT_REQUIRED else: self.sslcontext.check_hostname = False self.sslcontext.verify_mode = ssl.CERT_NONE self.test = test self.conn = None def do_select_mailbox(func): """ Decorator to do a fresh mailbox SELECT """ def wrapper(*args, **kwargs): if len(args) != 1: raise AttributeError( 'Size of *args tuple "{0}" isn\'t 1. It looks like you haven\'t specified all ' 'method arguments as named arguments!'.format(args)) mailbox = None for key in ['mailbox', 'source']: if key in kwargs.keys(): mailbox = kwargs[key] break if mailbox is None: raise KeyError( 'Unable to SELECT a mailbox, kwargs "{0}" doesn\'t contain a mailbox name' .format(kwargs)) result = args[0].select_mailbox(mailbox) if not result.code: raise RuntimeError(result.data) return func(*args, **kwargs) return wrapper def process_error(self, exception, simple_return=False): """ Process Python exception by logging a message and optionally showing traceback """ trace_info = exc_info() err_msg = str(exception) if isinstance(exception, IMAPClient.Error): err_msg = Helper().byte_to_str(exception.args[0]) self.logger.error("Catching IMAP exception {}: {}".format( type(exception), err_msg)) if self.logger.isEnabledFor(loglevel_DEBUG): print_exception(*trace_info) del trace_info if simple_return: return exception else: return self.Retval(False, err_msg) def connect(self, retry=True, logout=False): """ Connect to IMAP server and login """ if self.starttls: self.logger.debug( 'Establishing IMAP connection using STARTTLS/{} to {} and logging in with user {}' .format(self.port, self.server, self.username)) elif self.imaps: self.logger.debug( 'Establishing IMAP connection using SSL/{} (imaps) to {} and logging in with user {}' .format(self.port, self.server, self.username)) try: self.conn = IMAPClient(host=self.server, port=self.port, use_uid=True, ssl=self.imaps, ssl_context=self.sslcontext, timeout=self.timeout) if self.starttls: self.conn.starttls(ssl_context=self.sslcontext) login = self.conn.login(self.username, self.password) login_response = Helper().byte_to_str(login) # Test login/auth status login_success = False noop = self.noop() if noop.code and noop.data: login_success = True if logout: return self.disconnect() elif login_success: return self.Retval(True, login_response) else: return self.Retval(False, login_response) # pragma: no cover except exceptions.LoginError as e: return self.process_error(e) except Exception as e: err_return = self.process_error(e) if retry: self.logger.error('Trying one more time to login') sleep(2) return self.connect(retry=False, logout=logout) return err_return def noop(self): """ Do a noop to test login status """ try: noop = self.conn.noop() noop_response = Helper().byte_to_str(noop[0]) noop_resp_pattern_re = regex_compile('^(Success|NOOP completed)') login_success = noop_resp_pattern_re.match(noop_response) return self.Retval(True, login_success) except IMAPClient.Error as e: return self.process_error(e) def disconnect(self): """ Disconnect from IMAP server """ result = self.conn.logout() response = Helper().byte_to_str(result) return self.Retval(response == 'Logging out', response) def list_mailboxes(self, directory='', pattern='*'): """ Get a listing of folders (mailboxes) on the server """ try: raw_list = self.conn.list_folders(directory, pattern) nice_list = [] for mailbox in raw_list: flags = [] for flag in mailbox[0]: flags.append(flag.decode('utf-8')) nice_list.append({ 'name': mailbox[2], 'flags': flags, 'delimiter': mailbox[1].decode("utf-8") }) return self.Retval(True, nice_list) except IMAPClient.Error as e: return self.process_error(e) def select_mailbox(self, mailbox): """ Select a mailbox to work on """ self.logger.debug('Switching to mailbox {}'.format(mailbox)) try: result = self.conn.select_folder(mailbox) response = {} for key, value in result.items(): unicode_key = Helper().byte_to_str(key) if unicode_key == 'FLAGS': flags = [] for flag in value: flags.append(Helper().byte_to_str(flag)) response[unicode_key] = tuple(flags) else: response[unicode_key] = value return self.Retval(True, response) except IMAPClient.Error as e: return self.process_error(e) def add_mail(self, mailbox, message, flags=(), msg_time=None): """ Add/append a mail to a mailbox """ self.logger.debug('Adding a mail into mailbox {}'.format(mailbox)) try: if not isinstance(message, Mail): message = Mail(logger=self.logger, mail_native=message) self.conn.append(mailbox, str(message.get_native()), flags, msg_time) # According to rfc4315 we must not return the UID from the response, so we are fetching it ourselves uids = self.search_mails(mailbox=mailbox, criteria='HEADER Message-Id "{}"'.format( message.get_message_id())).data[0] return self.Retval(True, uids) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def search_mails(self, mailbox, criteria='ALL', autocreate_mailbox=False): """ Search for mails in a mailbox """ self.logger.debug( 'Searching for mails in mailbox {} and criteria=\'{}\''.format( mailbox, criteria)) try: return self.Retval(True, list(self.conn.search(criteria=criteria))) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def fetch_mails(self, uids, mailbox, return_fields=None): """ Retrieve mails from a mailbox """ self.logger.debug('Fetching mails with uids {}'.format(uids)) return_raw = True if return_fields is None: return_raw = False return_fields = [b'RFC822'] mails = {} try: for uid in uids: result = self.conn.fetch(uid, return_fields) if not result: continue if return_raw: mails[uid] = result[uid] else: # mails[uid] = Mail(logger=self.logger, uid=uid, mail_native=email.message_from_bytes(result[uid][b'RFC822'])) mails[uid] = Mail(logger=self.logger, mail_native=email.message_from_bytes( result[uid][b'RFC822'])) return self.Retval(True, mails) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def get_mailflags(self, uids, mailbox): """ Retrieve flags from mails """ try: result = self.conn.get_flags(uids) flags = {} for uid in uids: flags[uid] = [] if uid not in result.keys(): self.logger.error( 'Failed to get flags for mail with uid={}: {}'.format( uid, result)) return self.Retval(False, None) for flag in result[uid]: flags[uid].append(flag.decode('utf-8')) return self.Retval(True, flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def set_mailflags(self, uids, mailbox, flags=[]): """ Set and retrieve flags from mails """ if self.test: self.logger.info( 'Would have set mail flags on message uids "{}"'.format( str(uids))) return self.Retval(True, None) else: self.logger.debug('Setting flags={} on mails uid={}', flags, uids) try: result = self.conn.set_flags(uids, flags) _flags = {} for uid in uids: _flags[uid] = [] if uid not in result.keys(): self.logger.error( 'Failed to set and get flags for mail with uid={}: {}' .format(uid, result)) return self.Retval(False, None) for flag in result[uid]: _flags[uid].append(flag.decode('utf-8')) return self.Retval(True, _flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def add_mailflags(self, uids, mailbox, flags=[]): """ Add and retrieve flags from mails """ if self.test: self.logger.info( 'Would have added mail flags on message uids "{}"'.format( str(uids))) return self.Retval(True, None) else: self.logger.debug('Adding flags={} on mails uid={}', flags, uids) try: result = self.conn.add_flags(uids, flags) _flags = {} for uid in uids: _flags[uid] = [] if uid not in result.keys(): self.logger.error( 'Failed to add and get flags for mail with uid={}: {}' .format(uid, result)) return self.Retval(False, None) for flag in result[uid]: _flags[uid].append(flag.decode('utf-8')) return self.Retval(True, _flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def move_mail(self, message_ids, source, destination, delete_old=True, expunge=True, add_flags=None, set_flags=None): """ Move a mail from a mailbox to another """ return self.copy_mails(message_ids=message_ids, source=source, destination=destination, delete_old=delete_old, expunge=expunge, add_flags=add_flags, set_flags=set_flags) @do_select_mailbox def copy_mails(self, source, destination, message_ids=None, delete_old=False, expunge=False, add_flags=None, set_flags=None): """ Copies one or more mails from a mailbox into another """ if self.test: if delete_old: self.logger.info( 'Would have moved mail Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode' .format(message_ids, source, destination)) else: self.logger.info( 'Would have copied mails with Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode' .format(message_ids, source, destination)) return self.Retval(True, None) else: try: if delete_old: self.logger.debug( 'Moving mail Message-Ids="{}" from "{}" to "{}"'. format(message_ids, source, destination)) else: self.logger.debug( 'Copying mail Message-Ids="{}" from "{}" to "{}"'. format(message_ids, source, destination)) # if message_ids is None: # message_ids = [] # result = self.fetch_mails(uids=uids, mailbox=source) # if not result.code: # self.logger.error('Failed to determine Message-Id by uids for mail with uids "{}"', uids) # return result # message_ids.append(result.data.keys()) if not self.mailbox_exists(destination).data: self.logger.info( 'Destination mailbox {} doesn\'t exist, creating it for you' .format(destination)) result = self.create_mailbox(mailbox=destination) if not result.code: self.logger.error( 'Failed to create the mailbox {}: {}'.format( source, result.data)) # pragma: no cover return result # pragma: no cover uids = [] for message_id in message_ids: result = self.search_mails( mailbox=source, criteria='HEADER Message-Id "{}"'.format(message_id)) if not result.code or len(result.data) == 0: self.logger.error( 'Failed to determine uid by Message-Id for mail with Message-Id "{}"' .format(message_id)) return self.Retval(False, result.data) uids.append(result.data[0]) result = self.select_mailbox(source) if not result.code: return result # pragma: no cover self.conn.copy(uids, destination) if delete_old: result = self.delete_mails(uids=uids, mailbox=source) if not result.code: self.logger.error( 'Failed to remove old mail with Message-Id="{}"/uids="{}": {}' .format(message_ids, uids, result.data)) # pragma: no cover return result # pragma: no cover if expunge: # TODO don't expunge by default result = self.expunge(mailbox=source) if not result.code: self.logger.error( 'Failed to expunge on mailbox {}: {}'.format( source, result.data)) # pragma: no cover return result # pragma: no cover dest_uids = [] for message_id in message_ids: result = self.search_mails( mailbox=destination, criteria='HEADER Message-Id "{}"'.format(message_id)) if not result.code: self.logger.error( 'Failed to determine uid by Message-Id for mail with Message-Id "{}"' .format(message_id)) # pragma: no cover return result # pragma: no cover dest_uids.append(result.data[0]) if isinstance(set_flags, list): self.set_mailflags(uids=dest_uids, mailbox=destination, flags=set_flags) if add_flags: self.add_mailflags(uids=dest_uids, mailbox=destination, flags=add_flags) return self.Retval(True, dest_uids) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def expunge(self, mailbox): """ Expunge mails form a mailbox """ self.logger.debug('Expunge mails from mailbox {}'.format(mailbox)) try: return self.Retval(True, b'Expunge completed.' in self.conn.expunge()) except IMAPClient.Error as e: # pragma: no cover return self.process_error(e) # pragma: no cover def create_mailbox(self, mailbox): """ Create a mailbox """ self.logger.debug('Creating mailbox {}'.format(mailbox)) try: return self.Retval( True, self.conn.create_folder(mailbox) == b'Create completed.') except IMAPClient.Error as e: return self.process_error(e) def mailbox_exists(self, mailbox): """ Check whether a mailbox exists """ try: return self.Retval(True, self.conn.folder_exists(mailbox)) except IMAPClient.Error as e: # pragma: no cover return self.process_error(e) # pragma: no cover @do_select_mailbox def delete_mails(self, uids, mailbox): """ Delete mails """ self.logger.debug('Deleting mails with uid="{}"'.format(uids)) try: result = self.conn.delete_messages(uids) flags = {} for uid in uids: flags[uid] = [] if uid not in result.keys(): self.logger.error( 'Failed to get flags for mail with uid={} after deleting it: {}' .format(uid, result)) return self.Retval(False, None) for flag in result[uid]: flags[uid].append(flag.decode('utf-8')) return self.Retval(True, flags) except IMAPClient.Error as e: return self.process_error(e)
class Hauberk(): def __init__(self, dry_run=False): self.rules = list() self.dry = dry_run def login(self, username, password, server): logger.info("Connecting to Server") self.client = IMAPClient(server, use_uid=True) self.client.login(username, password) logger.info("Connected") def add_rule(self, filters, actions): self.rules.append(Rule(filters=filters, actions=actions)) if not filters: raise Exception("Must define filters in rule") if not actions: raise Exception("Must define actions in rule") logger.info("Added rule number %s", len(self.rules)) def run(self): messages = self.client.search() logger.info("%s messages to process", len(messages)) for msgid, data in self.client.fetch( messages, [FetchFlags.ENVELOPE.value, FetchFlags.BODY.value]).items(): envelope = data[FetchFlags.ENVELOPE.value] body = data[FetchFlags.BODY.value] message = Message(envelope, body) logger.debug("Processing message %s: %s", msgid, message.subject) done = False for rule in self.rules: for _filter in rule.filters: logger.debug("Running filter %s", _filter) if _filter(message): logger.debug("Match found!") # log here that there was a match for action in rule.actions: logger.debug("Running action %s", action) action(self.client, msgid, self.dry) done = True break if done: break def select_folder(self, folder): """Select working folder. :param folder: Folder rules should be applied to """ logger.info("Running in folder %s", folder) self.client.select_folder(folder) def existing_folders(self, folders): """Make sure certain folders exist client side. :param folders: List of folder names """ logger.info("Checking for existing folders") if not isinstance(folders, list): folders = [folders] for folder in folders: logger.debug("Checking for existance of folder %s" % folder) if not self.client.folder_exists(folder): logger.warning("Folder %s does not exist, creating" % folder) if not self.dry: self.client.create_folder(folder)
class ImapDB(BaseDB): def __init__(self, username, password='******', host='localhost', port=143, *args, **kwargs): super().__init__(username, *args, **kwargs) self.imap = IMAPClient(host, port, use_uid=True, ssl=False) res = self.imap.login(username, password) self.cursor.execute( "SELECT lowModSeq,highModSeq,highModSeqMailbox,highModSeqThread,highModSeqEmail FROM account LIMIT 1" ) row = self.cursor.fetchone() self.lastfoldersync = 0 if row: self.lowModSeq, self.highModSeq, self.highModSeqMailbox, self.highModSeqThread, self.highModSeqEmail = row else: self.lowModSeq = 0 self.highModSeq = 1 self.highModSeqMailbox = 1 self.highModSeqThread = 1 self.highModSeqEmail = 1 # (imapname, readonly) self.selected_folder = (None, False) self.mailboxes = {} self.sync_mailboxes() self.messages = {} def get_messages_cached(self, properties=(), id__in=()): messages = [] if not self.messages: return messages, id__in, properties fetch_props = set() fetch_ids = set(id__in) for id in id__in: msg = self.messages.get(id, None) if msg: found = True for prop in properties: try: msg[prop] except (KeyError, AttributeError): found = False fetch_props.add(prop) if found: fetch_ids.remove(id) messages.append(msg) # if one messages is missing, need to fetch all properties if len(messages) < len(id__in): fetch_props = properties return messages, fetch_ids, fetch_props def get_messages(self, properties=(), sort={}, inMailbox=None, inMailboxOtherThan=(), id__in=None, threadId__in=None, **criteria): # XXX: id == threadId for now if id__in is None and threadId__in is not None: id__in = [id[1:] for id in threadId__in] if id__in is None: messages = [] else: # try get everything from cache messages, id__in, properties = self.get_messages_cached( properties, id__in=id__in) fetch_fields = { f for prop, f in FIELDS_MAP.items() if prop in properties } if 'RFC822' in fetch_fields: # remove redundand fields fetch_fields.discard('RFC822.HEADER') fetch_fields.discard('RFC822.SIZE') if inMailbox: mailbox = self.mailboxes.get(inMailbox, None) if not mailbox: raise errors.notFound(f'Mailbox {inMailbox} not found') mailboxes = [mailbox] elif inMailboxOtherThan: mailboxes = [ m for m in self.mailboxes.values() if m['id'] not in inMailboxOtherThan ] else: mailboxes = self.mailboxes.values() search_criteria = as_imap_search(criteria) sort_criteria = as_imap_sort(sort) or '' if sort else None mailbox_uids = {} if id__in is not None: if len(id__in) == 0: return messages # no messages matches empty ids if not fetch_fields and not sort_criteria: # when we don't need anything new from IMAP, create empty messages # useful when requested conditions can be calculated from id (threadId) messages.extend( self.messages.get(id, 0) or ImapMessage(id=id) for id in id__in) return messages for id in id__in: # TODO: check uidvalidity mailboxid, uidvalidity, uid = parse_message_id(id) uids = mailbox_uids.get(mailboxid, []) if not uids: mailbox_uids[mailboxid] = uids uids.append(uid) # filter out unnecessary mailboxes mailboxes = [m for m in mailboxes if m['id'] in mailbox_uids] for mailbox in mailboxes: imapname = mailbox['imapname'] if self.selected_folder[0] != imapname: self.imap.select_folder(imapname, readonly=True) self.selected_folder = (imapname, True) uids = mailbox_uids.get(mailbox['id'], None) # uids are now None or not empty # fetch all if sort_criteria: if uids: search = f'{",".join(map(str, uids))} {search_criteria}' else: search = search_criteria or 'ALL' uids = self.imap.sort(sort_criteria, search) elif search_criteria: if uids: search = f'{",".join(map(str, uids))} {search_criteria}' uids = self.imap.search(search) if uids is None: uids = '1:*' fetch_fields.add('UID') fetches = self.imap.fetch(uids, fetch_fields) for uid, data in fetches.items(): id = format_message_id(mailbox['id'], mailbox['uidvalidity'], uid) msg = self.messages.get(id, None) if not msg: msg = ImapMessage(id=id, mailboxIds=[mailbox['id']]) self.messages[id] = msg for k, v in data.items(): msg[k.decode()] = v messages.append(msg) return messages def changed_record(self, ifolderid, uid, flags=(), labels=()): res = self.dmaybeupdate( 'imessages', { 'flags': json.dumps(sorted(flags)), 'labels': json.dumps(sorted(labels)), }, { 'ifolderid': ifolderid, 'uid': uid }) if res: msgid = self.dgefield('imessages', { 'ifolderid': ifolderid, 'uid': uid }, 'msgid') self.mark_sync(msgid) def import_message(self, rfc822, mailboxIds, keywords): folderdata = self.dget('ifolders') foldermap = {f['ifolderid']: f for f in folderdata} jmailmap = { f['jmailboxid']: f for f in folderdata if f.get('jmailboxid', False) } # store to the first named folder - we can use labels on gmail to add to other folders later. id, others = mailboxIds imapname = jmailmap[id][imapname] flags = set(keywords) for kw in flags: if kw in KEYWORD2FLAG: flags.remove(kw) flags.add(KEYWORD2FLAG[kw]) appendres = self.imap.append('imapname', '(' + ' '.join(flags) + ')', datetime.now(), rfc822) # TODO: compare appendres[2] with uidvalidity uid = appendres[3] fdata = jmailmap[mailboxIds[0]] self.do_folder(fdata['ifolderid'], fdata['label']) ifolderid = fdata['ifolderid'] msgdata = self.dgetone('imessages', { 'ifolderid': ifolderid, 'uid': uid, }, 'msgid,thrid,size') # XXX - did we fail to sync this back? Annoying if not msgdata: raise Exception( 'Failed to get back stored message from imap server') # save us having to download it again - drop out of transaction so we don't wait on the parse message = parse.parse(rfc822, msgdata['msgid']) self.begin() self.dinsert( 'jrawmessage', { 'msgid': msgdata['msgid'], 'parsed': json.dumps('message'), 'hasAttachment': message['hasattachment'], }) self.commit() return msgdata def update_messages(self, changes, idmap): if not changes: return {}, {} changed = {} notchanged = {} map = {} msgids = set(changes.keys()) sql = 'SELECT msgid,ifolderid,uid FROM imessages WHERE msgid IN (' + ( ('?,' * len(msgids))[:-1]) + ')' self.cursor.execute(sql, list(msgids)) for msgid, ifolderid, uid in self.cursor: if not msgid in map: map[msgid] = {ifolderid: {uid}} elif not ifolderid in map[msgid]: map[msgid][ifolderid] = {uid} else: map[msgid][ifolderid].add(uid) msgids.discard(msgid) for msgid in msgids: notchanged[msgid] = { 'type': 'notFound', 'description': 'No such message on server', } folderdata = self.dget('ifolders') foldermap = {f['ifolderid']: f for f in folderdata} jmailmap = { f['jmailboxid']: f for f in folderdata if 'jmailboxid' in f } jmapdata = self.dget('jmailboxes') jidmap = {d['jmailboxid']: (d['role'] or '') for d in jmapdata} jrolemap = { d['role']: d['jmailboxid'] for d in jmapdata if 'role' in d } for msgid in map.keys(): action = changes[msgid] try: for ifolderid, uids in map[msgid].items(): # TODO: merge similar actions? imapname = foldermap[ifolderid]['imapname'] uidvalidity = foldermap[ifolderid]['uidvalidity'] if self.selected_folder != (imapname, False): self.imap.select_folder(imapname) self.selected_folder = (imapname, False) if imapname and uidvalidity and 'keywords' in action: flags = set(action['keywords']) for kw in flags: if kw in KEYWORD2FLAG: flags.remove(kw) flags.add(KEYWORD2FLAG[kw]) self.imap.set_flags(uids, flags, silent=True) if 'mailboxIds' in action: mboxes = [idmap(k) for k in action['mailboxIds'].keys()] # existing ifolderids containing this message # identify a source message to work from ifolderid = sorted(map[msgid])[0] uid = sorted(map[msgid][ifolderid])[0] imapname = foldermap[ifolderid]['imapname'] uidvalidity = foldermap[ifolderid]['uidvalidity'] # existing ifolderids with this message current = set(map[msgid].keys()) # new ifolderids that should contain this message new = set(jmailmap[x]['ifolderid'] for x in mboxes) for ifolderid in new: # unless there's already a matching message in it if current.pop(ifolderid): continue # copy from the existing message newfolder = foldermap[ifolderid]['imapname'] self.imap.copy(imapname, uidvalidity, uid, newfolder) for ifolderid in current: # these ifolderids didn't exist in new, so delete all matching UIDs from these folders self.imap.move( foldermap[ifolderid]['imapname'], foldermap[ifolderid]['uidvalidity'], map[msgid][ifolderid], # uids ) except Exception as e: notchanged[msgid] = {'type': 'error', 'description': str(e)} raise e else: changed[msgid] = None return changed, notchanged def destroy_messages(self, ids): if not ids: return [], {} destroymap = defaultdict(dict) notdestroyed = {} idset = set(ids) rows = self.dget('imessages', {'msgid': ('IN', idset)}, 'msgid,ifolderid,uid') for msgid, ifolderid, uid in rows: idset.discard(msgid) destroymap[ifolderid][uid] = msgid for msgid in idset: notdestroyed[msgid] = { 'type': 'notFound', 'description': "No such message on server", } folderdata = self.dget('ifolders') foldermap = {d['ifolderid']: d for d in folderdata} jmailmap = { d['jmailboxid']: d for d in folderdata if 'jmailboxid' in d } destroyed = [] for ifolderid, ifolder in destroymap.items(): #TODO: merge similar actions? if not ifolder['imapname']: for msgid in destroymap[ifolderid]: notdestroyed[msgid] = \ {'type': 'notFound', 'description': "No folder"} self.imap.move(ifolder['imapname'], ifolder['uidvalidity'], destroymap[ifolderid].keys(), None) destroyed.extend(destroymap[ifolderid].values()) return destroyed, notdestroyed def deleted_record(self, ifolderid, uid): msgid = self.dgetfield('imessages', { 'ifolderid': ifolderid, 'uid': uid }, 'msgid') if msgid: self.ddelete('imessages', {'ifolderid': ifolderid, 'uid': uid}) self.mark_sync(msgid) def get_raw_message(self, msgid, part=None): self.cursor.execute( 'SELECT imapname,uidvalidity,uid FROM ifolders JOIN imessages USING (ifolderid) WHERE msgid=?', [msgid]) imapname, uidvalidity, uid = self.cursor.fetchone() if not imapname: return None typ = 'message/rfc822' if part: parsed = self.fill_messages([msgid]) typ = find_type(parsed[msgid], part) res = self.imap.getpart(imapname, uidvalidity, uid, part) return typ, res['data'] def get_mailboxes(self, fields=None, **criteria): byimapname = {} # TODO: LIST "" % RETURN (STATUS (UNSEEN MESSAGES HIGHESTMODSEQ MAILBOXID)) for flags, sep, imapname in self.imap.list_folders(): status = self.imap.folder_status(imapname, ([ 'MESSAGES', 'UIDVALIDITY', 'UIDNEXT', 'HIGHESTMODSEQ', 'X-GUID' ])) flags = [f.lower() for f in flags] roles = [f for f in flags if f not in KNOWN_SPECIALS] label = roles[0].decode() if roles else imapname role = ROLE_MAP.get(label.lower(), None) can_select = b'\\noselect' not in flags byimapname[imapname] = { # Dovecot can fetch X-GUID 'id': status[b'X-GUID'].decode(), 'parentId': None, 'name': imapname, 'role': role, 'sortOrder': 2 if role else (1 if role == 'inbox' else 3), 'isSubscribed': True, # TODO: use LSUB 'totalEmails': status[b'MESSAGES'], 'unreadEmails': 0, 'totalThreads': 0, 'unreadThreads': 0, 'myRights': { 'mayReadItems': can_select, 'mayAddItems': can_select, 'mayRemoveItems': can_select, 'maySetSeen': can_select, 'maySetKeywords': can_select, 'mayCreateChild': True, 'mayRename': False if role else True, 'mayDelete': False if role else True, 'maySubmit': can_select, }, 'imapname': imapname, 'sep': sep.decode(), 'uidvalidity': status[b'UIDVALIDITY'], 'uidnext': status[b'UIDNEXT'], # Data sync properties 'createdModSeq': status[b'UIDVALIDITY'], # TODO: persist 'updatedModSeq': status[b'UIDVALIDITY'], # TODO: from persistent storage 'updatedNotCountsModSeq': status[b'UIDVALIDITY'], # TODO: from persistent storage 'emailHighestModSeq': status[b'HIGHESTMODSEQ'], 'deleted': 0, } # set name and parentId for child folders for imapname, mailbox in byimapname.items(): names = imapname.rsplit(mailbox['sep'], maxsplit=1) if len(names) == 2: mailbox['parentId'] = byimapname[names[0]]['id'] mailbox['name'] = names[1] # update cache self.mailboxes = {mbox['id']: mbox for mbox in byimapname.values()} return byimapname.values() def sync_mailboxes(self): self.get_mailboxes() def mailbox_imapname(self, parentId, name): parent = self.mailboxes.get(parentId, None) if not parent: raise errors.notFound('parent folder not found') return parent['imapname'] + parent['sep'] + name def create_mailbox(self, name=None, parentId=None, isSubscribed=True, **kwargs): if not name: raise errors.invalidProperties('name is required') imapname = self.mailbox_imapname(parentId, name) # TODO: parse returned MAILBOXID try: res = self.imap.create_folder(imapname) except IMAPClientError as e: desc = str(e) if '[ALREADYEXISTS]' in desc: raise errors.invalidArguments(desc) except Exception: raise errors.serverFail(res.decode()) if not isSubscribed: self.imap.unsubscribe_folder(imapname) status = self.imap.folder_status(imapname, ['UIDVALIDITY']) self.sync_mailboxes() return f"f{status[b'UIDVALIDITY']}" def update_mailbox(self, id, name=None, parentId=None, isSubscribed=None, sortOrder=None, **update): mailbox = self.mailboxes.get(id, None) if not mailbox: raise errors.notFound('mailbox not found') imapname = mailbox['imapname'] if (name is not None and name != mailbox['name']) or \ (parentId is not None and parentId != mailbox['parentId']): if not name: raise errors.invalidProperties('name is required') newimapname = self.mailbox_imapname(parentId, name) res = self.imap.rename_folder(imapname, newimapname) if b'NO' in res or b'BAD' in res: raise errors.serverFail(res.encode()) if isSubscribed is not None and isSubscribed != mailbox['isSubscribed']: if isSubscribed: res = self.imap.subscribe_folder(imapname) else: res = self.imap.unsubscribe_folder(imapname) if b'NO' in res or b'BAD' in res: raise errors.serverFail(res.encode()) if sortOrder is not None and sortOrder != mailbox['sortOrder']: # TODO: update in persistent storage mailbox['sortOrder'] = sortOrder self.sync_mailboxes() def destroy_mailbox(self, id): mailbox = self.mailboxes.get(id, None) if not mailbox: raise errors.notFound('mailbox not found') res = self.imap.delete_folder(mailbox['imapname']) if b'NO' in res or b'BAD' in res: raise errors.serverFail(res.encode()) mailbox['deleted'] = datetime.now().timestamp() self.sync_mailboxes() def create_submission(self, new, idmap): if not new: return {}, {} todo = {} createmap = {} notcreated = {} for cid, sub in new.items(): msgid = idmap.get(sub['emailId'], sub['emailId']) if not msgid: notcreated[cid] = {'error': 'nos msgid provided'} continue thrid = self.dgetfield('jmessages', { 'msgid': msgid, 'deleted': 0 }, 'thrid') if not thrid: notcreated[cid] = {'error': 'message does not exist'} continue id = self.dmake( 'jsubmission', { 'sendat': datetime.fromisoformat()(sub['sendAt']).isoformat() if sub['sendAt'] else datetime.now().isoformat(), 'msgid': msgid, 'thrid': thrid, 'envelope': json.dumps(sub['envelope']) if 'envelope' in sub else None, }) createmap[cid] = {'id': id} todo[cid] = msgid self.commit() for cid, sub in todo.items(): type, rfc822 = self.get_raw_message(todo[cid]) self.imap.send_mail(rfc822, sub['envelope']) return createmap, notcreated def update_submission(self, changed, idmap): return {}, {x: 'change not supported' for x in changed.keys()} def destroy_submission(self, destroy): if not destroy: return [], {} destroyed = [] notdestroyed = {} namemap = {} for subid in destroy: deleted = self.dgetfield('jsubmission', {'jsubid': subid}, 'deleted') if deleted: destroy.append(subid) self.ddelete('jsubmission', {'jsubid': subid}) else: notdestroyed[subid] = { 'type': 'notFound', 'description': 'submission not found' } self.commit() return destroyed, notdestroyed def _initdb(self): super()._initdb() self.dbh.execute(""" CREATE TABLE IF NOT EXISTS ifolders ( ifolderid INTEGER PRIMARY KEY NOT NULL, jmailboxid INTEGER, sep TEXT NOT NULL, imapname TEXT NOT NULL, label TEXT, uidvalidity INTEGER, uidfirst INTEGER, uidnext INTEGER, highestmodseq INTEGER, uniqueid TEXT, mtime DATE NOT NULL )""") self.dbh.execute( "CREATE INDEX IF NOT EXISTS ifolderj ON ifolders (jmailboxid)") self.dbh.execute( "CREATE INDEX IF NOT EXISTS ifolderlabel ON ifolders (label)") self.dbh.execute(""" CREATE TABLE IF NOT EXISTS imessages ( imessageid INTEGER PRIMARY KEY NOT NULL, ifolderid INTEGER, uid INTEGER, internaldate DATE, modseq INTEGER, flags TEXT, labels TEXT, thrid TEXT, msgid TEXT, envelope TEXT, bodystructure TEXT, size INTEGER, mtime DATE NOT NULL )""") self.dbh.execute( "CREATE UNIQUE INDEX IF NOT EXISTS imsgfrom ON imessages (ifolderid, uid)" ) self.dbh.execute( "CREATE INDEX IF NOT EXISTS imessageid ON imessages (msgid)") self.dbh.execute( "CREATE INDEX IF NOT EXISTS imessagethrid ON imessages (thrid)") self.dbh.execute(""" CREATE TABLE IF NOT EXISTS ithread ( messageid TEXT PRIMARY KEY, sortsubject TEXT, thrid TEXT )""") self.dbh.execute( "CREATE INDEX IF NOT EXISTS ithrid ON ithread (thrid)")
class IMAP(): """ Central class for IMAP server communication """ Retval = namedtuple('Retval', 'code data') def __init__(self, logger, username, password, server='localhost', port=143, starttls=False, imaps=False, tlsverify=True, test=False, timeout=None): self.logger = logger self.username = username self.password = password self.server = server self.port = port self.imaps = imaps self.starttls = starttls self.timeout = timeout self.sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) # TODO add proto arg if tlsverify: self.sslcontext.verify_mode = ssl.CERT_REQUIRED else: self.sslcontext.verify_mode = ssl.CERT_NONE # TODO improve? self.test = test self.conn = None def do_select_mailbox(func): """ Decorator to do a fresh mailbox SELECT """ def wrapper(*args, **kwargs): if len(args) != 1: raise AttributeError( 'Size of *args tuple "{0}" isn\'t 1. It looks like you haven\'t specified all ' 'method arguments as named arguments!'.format( args)) mailbox = None for key in ['mailbox', 'source']: if key in kwargs.keys(): mailbox = kwargs[key] break if mailbox is None: raise KeyError('Unable to SELECT a mailbox, kwargs "{0}" doesn\'t contain a mailbox name'.format(kwargs)) result = args[0].select_mailbox(mailbox) if not result.code: raise RuntimeError(result.data) return func(*args, **kwargs) return wrapper def process_error(self, exception, simple_return=False): """ Process Python exception by logging a message and optionally showing traceback """ trace_info = exc_info() err_msg = str(exception) if isinstance(exception, IMAPClient.Error): err_msg = Helper().byte_to_str(exception.args[0]) self.logger.error('Catching IMAP exception %s: %s', type(exception), err_msg) if self.logger.isEnabledFor(loglevel_DEBUG): print_exception(*trace_info) del trace_info if simple_return: return exception else: return self.Retval(False, err_msg) def connect(self, retry=True, logout=False): """ Connect to IMAP server and login """ if self.starttls: self.logger.debug('Establishing IMAP connection using STARTTLS/%s to %s and logging in with user %s', self.port, self.server, self.username) elif self.imaps: self.logger.debug('Establishing IMAP connection using SSL/%s (imaps) to %s and logging in with user %s', self.port, self.server, self.username) login = '' err_return = None try: self.conn = IMAPClient(host=self.server, port=self.port, use_uid=True, ssl=self.imaps, ssl_context=self.sslcontext, timeout=self.timeout) if self.starttls: self.conn.starttls(ssl_context=self.sslcontext) login = self.conn.login(self.username, self.password) login_response = Helper().byte_to_str(login) # Test login/auth status login_success = False noop = self.noop() if noop.code and noop.data: login_success = True if logout: return self.disconnect() elif login_success: return self.Retval(True, login_response) else: return self.Retval(False, login_response) # pragma: no cover except Exception as e: err_return = self.process_error(e) if err_return.data == '[AUTHENTICATIONFAILED] Authentication failed.': return err_return if retry: self.logger.error('Trying one more time to login') sleep(2) return self.connect(retry=False, logout=logout) return err_return def noop(self): """ Do a noop to test login status """ try: noop = self.conn.noop() noop_response = Helper().byte_to_str(noop[0]) noop_resp_pattern_re = regex_compile('^(Success|NOOP completed)') login_success = noop_resp_pattern_re.match(noop_response) return self.Retval(True, login_success) except IMAPClient.Error as e: return self.process_error(e) def disconnect(self): """ Disconnect from IMAP server """ result = self.conn.logout() response = Helper().byte_to_str(result) return self.Retval(response == 'Logging out', response) def list_mailboxes(self, directory='', pattern='*'): """ Get a listing of folders (mailboxes) on the server """ try: raw_list = self.conn.list_folders(directory, pattern) nice_list = [] for mailbox in raw_list: flags = [] for flag in mailbox[0]: flags.append(flag.decode('utf-8')) nice_list.append({'name': mailbox[2], 'flags': flags, 'delimiter': mailbox[1].decode("utf-8")}) return self.Retval(True, nice_list) except IMAPClient.Error as e: return self.process_error(e) def select_mailbox(self, mailbox): """ Select a mailbox to work on """ self.logger.debug('Switching to mailbox %s', mailbox) try: result = self.conn.select_folder(mailbox) response = {} for key, value in result.items(): unicode_key = Helper().byte_to_str(key) if unicode_key == 'FLAGS': flags = [] for flag in value: flags.append(Helper().byte_to_str(flag)) response[unicode_key] = tuple(flags) else: response[unicode_key] = value return self.Retval(True, response) except IMAPClient.Error as e: return self.process_error(e) def add_mail(self, mailbox, message, flags=(), msg_time=None): """ Add/append a mail to a mailbox """ self.logger.debug('Adding a mail into mailbox %s', mailbox) try: if not isinstance(message, Mail): message = Mail(logger=self.logger, mail_native=message) message_native = message.get_native() #self.conn.append(mailbox, message, flags, msg_time) self._append(mailbox, str(message_native), flags, msg_time) # According to rfc4315 we must not return the UID from the response, so we are fetching it ourselves uids = self.search_mails(mailbox=mailbox, criteria='HEADER Message-Id "{0}"'.format(message.get_header('Message-Id'))).data[0] return self.Retval(True, uids) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def search_mails(self, mailbox, criteria='ALL', autocreate_mailbox=False): """ Search for mails in a mailbox """ self.logger.debug('Searching for mails in mailbox %s and criteria=\'%s\'', mailbox, criteria) try: return self.Retval(True, list(self.conn.search(criteria=criteria))) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def fetch_mails(self, uids, mailbox, return_fields=None): """ Retrieve mails from a mailbox """ self.logger.debug('Fetching mails with uids %s', uids) return_raw = True if return_fields is None: return_raw = False return_fields = [b'RFC822'] mails = {} try: for uid in uids: result = self.conn.fetch(uid, return_fields) if not result: continue if return_raw: mails[uid] = result[uid] else: #mails[uid] = Mail(logger=self.logger, uid=uid, mail_native=email.message_from_bytes(result[uid][b'RFC822'])) mails[uid] = Mail(logger=self.logger, mail_native=email.message_from_bytes(result[uid][b'RFC822'])) return self.Retval(True, mails) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def get_mailflags(self, uids, mailbox): """ Retrieve flags from mails """ try: result = self.conn.get_flags(uids) flags = {} for uid in uids: flags[uid] = [] if uid not in result.keys(): self.logger.error('Failed to get flags for mail with uid=%s: %s', uid, result) return self.Retval(False, None) for flag in result[uid]: flags[uid].append(flag.decode('utf-8')) return self.Retval(True, flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def set_mailflags(self, uids, mailbox, flags=[]): """ Set and retrieve flags from mails """ if self.test: self.logger.info('Would have set mail flags on message uids "%s"', str(uids)) return self.Retval(True, None) else: self.logger.debug('Setting flags=%s on mails uid=%s', flags, uids) try: result = self.conn.set_flags(uids, flags) _flags = {} for uid in uids: _flags[uid] = [] if uid not in result.keys(): self.logger.error('Failed to set and get flags for mail with uid=%s: %s', uid, result) return self.Retval(False, None) for flag in result[uid]: _flags[uid].append(flag.decode('utf-8')) return self.Retval(True, _flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def add_mailflags(self, uids, mailbox, flags=[]): """ Add and retrieve flags from mails """ if self.test: self.logger.info('Would have added mail flags on message uids "%s"', str(uids)) return self.Retval(True, None) else: self.logger.debug('Adding flags=%s on mails uid=%s', flags, uids) try: result = self.conn.add_flags(uids, flags) _flags = {} for uid in uids: _flags[uid] = [] if uid not in result.keys(): self.logger.error('Failed to add and get flags for mail with uid=%s: %s', uid, result) return self.Retval(False, None) for flag in result[uid]: _flags[uid].append(flag.decode('utf-8')) return self.Retval(True, _flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def move_mail(self, message_ids, source, destination, delete_old=True, expunge=True, add_flags=None, set_flags=None): """ Move a mail from a mailbox to another """ return self.copy_mails(message_ids=message_ids, source=source, destination=destination, delete_old=delete_old, expunge=expunge, add_flags=add_flags, set_flags=set_flags) @do_select_mailbox def copy_mails(self, source, destination, message_ids=None, delete_old=False, expunge=False, add_flags=None, set_flags=None): """ Copies one or more mails from a mailbox into another """ if self.test: if delete_old: self.logger.info('Would have moved mail Message-Ids="%s" from "%s" to "%s", skipping because of beeing in testmode', message_ids, source, destination) else: self.logger.info('Would have copied mails with Message-Ids="%s" from "%s" to "%s", skipping because of beeing in testmode', message_ids, source, destination) return self.Retval(True, None) else: try: if delete_old: self.logger.debug('Moving mail Message-Ids="%s" from "%s" to "%s"', message_ids, source, destination) else: self.logger.debug('Copying mail Message-Ids="%s" from "%s" to "%s"', message_ids, source, destination) #if message_ids is None: # message_ids = [] # result = self.fetch_mails(uids=uids, mailbox=source) # if not result.code: # self.logger.error('Failed to determine Message-Id by uids for mail with uids "%s"', uids) # return result # message_ids.append(result.data.keys()) if not self.mailbox_exists(destination).data: self.logger.info('Destination mailbox %s doesn\'t exist, creating it for you', destination) result = self.create_mailbox(mailbox=destination) if not result.code: self.logger.error('Failed to create the mailbox %s: %s', source, result.data) # pragma: no cover return result # pragma: no cover uids = [] for message_id in message_ids: result = self.search_mails(mailbox=source, criteria='HEADER Message-Id "{0}"'.format(message_id)) if not result.code or len(result.data) == 0: self.logger.error('Failed to determine uid by Message-Id for mail with Message-Id "%s"', message_id) return self.Retval(False, result.data) uids.append(result.data[0]) result = self.select_mailbox(source) if not result.code: return result # pragma: no cover self.conn.copy(uids, destination) if delete_old: result = self.delete_mails(uids=uids, mailbox=source) if not result.code: self.logger.error('Failed to remove old mail with Message-Id="%s"/uids="%s": %s', message_ids, uids, result.data) # pragma: no cover return result # pragma: no cover if expunge: # TODO don't expunge by default result = self.expunge(mailbox=source) if not result.code: self.logger.error('Failed to expunge on mailbox %s: %s', source, result.data) # pragma: no cover return result # pragma: no cover dest_uids = [] for message_id in message_ids: result = self.search_mails(mailbox=destination, criteria='HEADER Message-Id "{0}"'.format(message_id)) if not result.code: self.logger.error('Failed to determine uid by Message-Id for mail with Message-Id "%s"', message_id) # pragma: no cover return result # pragma: no cover dest_uids.append(result.data[0]) if isinstance(set_flags, list): self.set_mailflags(uids=dest_uids, mailbox=destination, flags=set_flags) if add_flags: self.add_mailflags(uids=dest_uids, mailbox=destination, flags=add_flags) return self.Retval(True, dest_uids) except IMAPClient.Error as e: return self.process_error(e) def _append(self, folder, msg, flags=(), msg_time=None): # TODO """ FORKED FORM IMAPCLIENT """ if msg_time: if not msg_time.tzinfo: # pragma: no cover msg_time = msg_time.replace(tzinfo=FixedOffset.for_system()) # pragma: no cover time_val = '"{0}"'.format(msg_time.strftime("%d-%b-%Y %H:%M:%S %z")) time_val = imapclient.imapclient.to_unicode(time_val) else: time_val = None return self.conn._command_and_check('append', self.conn._normalise_folder(folder), imapclient.imapclient.seq_to_parenstr(flags), time_val, Helper.str_to_bytes(msg), unpack=True) @do_select_mailbox def expunge(self, mailbox): """ Expunge mails form a mailbox """ self.logger.debug('Expunge mails from mailbox %s', mailbox) try: return self.Retval(True, b'Expunge completed.' in self.conn.expunge()) except IMAPClient.Error as e: # pragma: no cover return self.process_error(e) # pragma: no cover def create_mailbox(self, mailbox): """ Create a mailbox """ self.logger.debug('Creating mailbox %s', mailbox) try: return self.Retval(True, self.conn.create_folder(mailbox) == b'Create completed.') except IMAPClient.Error as e: return self.process_error(e) def mailbox_exists(self, mailbox): """ Check whether a mailbox exists """ try: return self.Retval(True, self.conn.folder_exists(mailbox)) except IMAPClient.Error as e: # pragma: no cover return self.process_error(e) # pragma: no cover @do_select_mailbox def delete_mails(self, uids, mailbox): """ Delete mails """ self.logger.debug('Deleting mails with uid="%s"', uids) try: result = self.conn.delete_messages(uids) flags = {} for uid in uids: flags[uid] = [] if uid not in result.keys(): self.logger.error('Failed to get flags for mail with uid=%s after deleting it: %s', uid, result) return self.Retval(False, None) for flag in result[uid]: flags[uid].append(flag.decode('utf-8')) return self.Retval(True, flags) except IMAPClient.Error as e: return self.process_error(e)
class MailFetcher(object): def __init__(self, host, user, password, folder_inbox, folder_archive, folder_error): self.server = IMAPClient(host) self.server.login(user, password) self.folder_inbox = folder_inbox self.folder_archive = folder_archive self.folder_error = folder_error if self.server.folder_exists(self.folder_archive): logger.info("Archive folder '{folder}' exists.".format( folder=self.folder_archive)) else: self.server.create_folder(self.folder_archive) logger.info("Archive folder '{folder}' created.".format( folder=self.folder_archive)) if self.server.folder_exists(self.folder_error): logger.info("Error folder '{folder}' exists.".format( folder=self.folder_error)) else: self.server.create_folder(self.folder_error) logger.info("Error folder '{folder}' created.".format( folder=self.folder_error)) def findIcs(self, rawmail): email_message = email.message_from_bytes(rawmail) for part in email_message.walk(): logger.debug(part.get_content_type()) if ("text/calendar" in part.get_content_type()): logger.info( "Found calendar event in mail from {sender} with subject '{subject}'." .format(sender=email_message.get('From'), subject=email_message.get('Subject'))) return ical_tools.parse(part.get_payload(decode=True)) def getIcs(self): select_info = self.server.select_folder(self.folder_inbox) logger.debug("{num_mails} messages in folder {folder}.".format( num_mails=select_info[b'EXISTS'], folder=self.folder_inbox)) ics_list = [] messages = self.server.search('ALL') for message_uid, message_data in self.server.fetch(messages, 'RFC822').items(): logger.debug("Found message with id {id}".format(id=message_uid)) ics = self.findIcs(message_data[b'RFC822']) if (ics): ics_list.append((message_uid, ics)) return ics_list def archiveMessage(self, uid, error=False): folder = self.folder_archive if error: folder = self.folder_error self.server.move([uid], folder) logger.info("Message with UID {uid} moved to folder {folder}".format( uid=uid, folder=folder))
class EmailListener: """EmailListener object for listening to an email folder and processing emails. Attributes: email (str): The email to listen to. app_password (str): The password for the email. folder (str): The email folder to listen in. attachment_dir (str): The file path to the folder to save scraped emails and attachments to. server (IMAPClient): The IMAP server to log into. Defaults to None. """ def __init__(self, email, app_password, folder, attachment_dir): """Initialize an EmailListener instance. Args: email (str): The email to listen to. app_password (str): The password for the email. folder (str): The email folder to listen in. attachment_dir (str): The file path to folder to save scraped emails and attachments to. Returns: None """ self.email = email self.app_password = app_password self.folder = folder self.attachment_dir = attachment_dir self.server = None def login(self): """Logs in the EmailListener to the IMAP server. Args: None Returns: None """ self.server = IMAPClient('imap.gmail.com') self.server.login(self.email, self.app_password) self.server.select_folder(self.folder, readonly=False) def logout(self): """Logs out the EmailListener from the IMAP server. Args: None Returns: None """ self.server.logout() self.server = None def scrape(self, move=None, unread=False, delete=False): """Scrape unread emails from the current folder. Args: move (str): The folder to move the emails to. If None, the emails are not moved. Defaults to None. unread (bool): Whether the emails should be marked as unread. Defaults to False. delete (bool): Whether the emails should be deleted. Defaults to False. Returns: A list of the file paths to each scraped email. """ # Ensure server is connected if type(self.server) is not IMAPClient: raise ValueError("server attribute must be type IMAPClient") # List containing the file paths of each file created for an email message msg_dict = {} # Search for unseen messages messages = self.server.search("UNSEEN") # For each unseen message for uid, message_data in self.server.fetch(messages, 'RFC822').items(): # Get the message email_message = email.message_from_bytes(message_data[b'RFC822']) # Get who the message is from from_email = self.__get_from(email_message) # Generate the dict key for this email key = "{}".format(uid) # Generate the value dictionary to be filled later val_dict = {} email_directory = os.path.join(self.attachment_dir, from_email, str(uid)) val_dict["email_directory"] = email_directory val_dict["from_email"] = from_email if not os.path.exists(email_directory): os.makedirs(email_directory) # Display notice print("PROCESSING: Email UID = {} from {}".format(uid, from_email)) # Add the subject val_dict["Subject"] = self.__get_subject(email_message).strip() # If the email has multiple parts if email_message.is_multipart(): val_dict = self.__parse_multipart_message( email_message, val_dict) # If the message isn't multipart else: val_dict = self.__parse_singlepart_message( email_message, val_dict) msg_dict[key] = val_dict # If required, move the email, mark it as unread, or delete it self.__execute_options(uid, move, unread, delete) # Return the dictionary of messages and their contents return msg_dict def __get_from(self, email_message): """Helper function for getting who an email message is from. Args: email_message (email.message): The email message to get sender of. Returns: A string containing the from email address. """ from_raw = email_message.get_all('From', []) from_list = email.utils.getaddresses(from_raw) if len(from_list[0]) == 1: from_email = from_list[0][0] elif len(from_list[0]) == 2: from_email = from_list[0][1] else: from_email = "UnknownEmail" return from_email def __get_subject(self, email_message): """ """ # Get the subject subject = email_message.get("Subject") # If there isn't a subject if subject is None: return "No Subject" return subject def __parse_multipart_message(self, email_message, val_dict): """Helper function for parsing multipart email messages. Args: email_message (email.message): The email message to parse. val_dict (dict): A dictionary containing the message data from each part of the message. Will be returned after it is updated. Returns: The dictionary containing the message data for each part of the message. """ # For each part for part in email_message.walk(): # If the part is an attachment file_name = part.get_filename() if bool(file_name): # Generate file path file_path = os.path.join(val_dict["email_directory"], encoded_words_to_text(file_name)) file = open(file_path, 'wb') file.write(part.get_payload(decode=True)) file.close() # Get the list of attachments, or initialize it if there isn't one attachment_list = val_dict.get("attachments") or [] attachment_list.append("{}".format(file_path)) val_dict["attachments"] = attachment_list # If the part is html text elif part.get_content_type() == 'text/html': # Convert the body from html to plain text val_dict["Plain_HTML"] = html2text.html2text( part.get_payload()) val_dict["HTML"] = part.get_payload() # If the part is plain text elif part.get_content_type() == 'text/plain': # Get the body val_dict["Plain_Text"] = part.get_payload() return val_dict def __parse_singlepart_message(self, email_message, val_dict): """Helper function for parsing singlepart email messages. Args: email_message (email.message): The email message to parse. val_dict (dict): A dictionary containing the message data from each part of the message. Will be returned after it is updated. Returns: The dictionary containing the message data for each part of the message. """ # Get the message body, which is plain text val_dict["Plain_Text"] = email_message.get_payload() return val_dict def __execute_options(self, uid, move, unread, delete): """Loop through optional arguments and execute any required processing. Args: uid (int): The email ID to process. move (str): The folder to move the emails to. If None, the emails are not moved. Defaults to None. unread (bool): Whether the emails should be marked as unread. Defaults to False. delete (bool): Whether the emails should be deleted. Defaults to False. Returns: None """ # If the message should be marked as unread if bool(unread): self.server.remove_flags(uid, [SEEN]) # If a move folder is specified if move is not None: try: # Move the message to another folder self.server.move(uid, move) except: # Create the folder and move the message to the folder self.server.create_folder(move) self.server.move(uid, move) # If the message should be deleted elif bool(delete): # Move the email to the trash self.server.set_gmail_labels(uid, "\\Trash") return def listen(self, process_func=write_txt_file, **kwargs): """Listen in an email folder for incoming emails, and process them. Args: process_func (function): A function called to further process the emails. The function must take only the list of file paths returned by the scrape function as an argument. Defaults to the example function write_txt_file in the email_processing module. **kwargs (dict): Additional arguments for processing the email. Optional arguments include: move (str): The folder to move emails to. If not set, the emails will not be moved. unread (bool): Whether the emails should be marked as unread. If not set, emails are kept as read. delete (bool): Whether the emails should be deleted. If not set, emails are not deleted. Returns: None """ # Ensure server is connected if type(self.server) is not IMAPClient: raise ValueError("server attribute must be type IMAPClient") while True: self.__idle(process_func=process_func, **kwargs) return def __idle(self, process_func=write_txt_file, **kwargs): """Helper function, idles in an email folder processing incoming emails. Args: process_func (function): A function called to further process the emails. The function must take only the list of file paths returned by the scrape function as an argument. Defaults to the example function write_txt_file in the email_processing module. **kwargs (dict): Additional arguments for processing the email. Optional arguments include: move (str): The folder to move emails to. If not set, the emails will not be moved. unread (bool): Whether the emails should be marked as unread. If not set, emails are kept as read. delete (bool): Whether the emails should be deleted. If not set, emails are not deleted. Returns: None """ # Set the relevant kwarg variables move = kwargs.get('move') unread = bool(kwargs.get('unread')) delete = bool(kwargs.get('delete')) # Start idling self.server.idle() # Set idle timeout to 5 minutes inner_timeout = get_time() + 60 * 5 # Until idle times out while get_time() < inner_timeout: # Check for a new response every 30 seconds responses = self.server.idle_check(timeout=30) # If there is a response if responses: print("Server sent:", responses) # Suspend the idling self.server.idle_done() # Process the new emails msgs = self.scrape(move=move, unread=unread, delete=delete) # Run the process function process_func(msgs) # Restart idling self.server.idle() # Stop idling self.server.idle_done() return
class gmailPy(object): def __init__(self): self.IMAP_SERVER = 'imap.gmail.com' self.ssl = True self.myIMAPc = None self.response = None self.folders = [] def login(self, username, password): self.myIMAPc = IMAPClient(self.IMAP_SERVER, ssl=self.ssl) self.myIMAPc.login(username, password) # Returns a list of all the folders for a particular account def get_folders(self): self.response = self.myIMAPc.list_folders() for item in self.response: self.folders.append(item[2].strip('u')) return self.folders # Returns the total number of messages in a folder def get_mail_count(self, folder='Inbox'): self.response = self.myIMAPc.select_folder(folder, True) return self.response['EXISTS'] # Method to delete messages based on their size def delete_bigmail(self, folder='Inbox'): self.myIMAPc.select_folder(folder, False) # Gets all the message ids of the messages which are not deleted in the folder messages = self.myIMAPc.search(['NOT DELETED']) print "%d messages that aren't deleted" % len(messages) if len(messages) > 0: print "You can exit by entering 0 or pressing CTRL+C \n" else: print "There are no messages in the folder" # Gets the message sizes for all the message ids returned in previous step # Note: Just sends one request for all message ids with a return time < 10 ms self.response = self.myIMAPc.fetch(messages, ['RFC822.SIZE']) # Sorts the dictionary returned by fetch by size in descending order sorted_response = sorted(self.response.iteritems(), key=operator.itemgetter(1), reverse=True) count = 1 try: for item in sorted_response: # Gets the biggest message including headers, body, etc. big_message = self.myIMAPc.fetch(item[0], ['RFC822']) for msgid, data in big_message.iteritems(): msg_string = data['RFC822'] # Parses the message string using email library msg = email.message_from_string(msg_string) val = dict(self.response[msgid])['RFC822.SIZE'] print 'ID %d: From: %s Date: %s' % (msgid, msg['From'], msg['date']) print 'To: %s' % (msg['To']) print 'Subject: %s' % (msg['Subject']) print 'Size: %d bytes \n' % (val) user_del = raw_input("Do you want to delete this message?(Y/N): ") if user_del == 'Y': self.delete_message(msgid) if count == len(sorted_response): print "There are no more messages" else: print "\nMoving on to the next biggest message >>> \n" elif user_del == '0': print "Program exiting" sys.exit() else: if count == len(sorted_response): print "There are no more messages" else: print "\nMoving on to the next biggest message >>> \n" count += 1 except KeyboardInterrupt: print "Program exiting" sys.exit() # Method to delete messages based on their size with a search criteria def delete_bigmail_search(self, folder='Inbox', command='', criteria=''): self.myIMAPc.select_folder(folder, False) # Gets all the message ids from the server based on the search criteria messages = self.myIMAPc.search('%s "%s"' % (command, criteria)) print "%d messages that match --> %s: %s" % (len(messages), command, criteria) if len(messages) > 0: print "You can exit by entering 0 or pressing CTRL+C \n" else: print "There are no messages in that matched your search criteria" # Gets the message sizes for all the message ids returned in previous step # Note: Just sends one request for all message ids with a return time < 10 ms self.response = self.myIMAPc.fetch(messages, ['RFC822.SIZE']) # Sorts the messages in decending order of their sizes sorted_response = sorted(self.response.iteritems(), key=operator.itemgetter(1), reverse=True) count = 1 try: for item in sorted_response: # Gets the entire content for the biggest message identified big_message = self.myIMAPc.fetch(item[0], ['RFC822']) for msgid, data in big_message.iteritems(): msg_string = data['RFC822'] msg = email.message_from_string(msg_string) val = dict(self.response[msgid])['RFC822.SIZE'] print 'ID %d: From: %s Date: %s' % (msgid, msg['From'], msg['date']) print 'To: %s' % (msg['To']) print 'Subject: %s' % (msg['Subject']) print 'Size: %d bytes \n' % (val) user_del = raw_input("Do you want to delete this message?(Y/N): ") if user_del == 'Y': self.delete_message(msgid) if count == len(sorted_response): print "There are no more messages" else: print "\nMoving on to the next biggest message >>> \n" elif user_del == '0': print "Program exiting" sys.exit() else: if count == len(sorted_response): print "There are no more messages" else: print "\nMoving on to the next biggest message >>> \n" count += 1 except KeyboardInterrupt: print "Program exiting" sys.exit() # Deletes a message in the current folder based on msg id def delete_message(self, id): try: self.myIMAPc.delete_messages([id]) self.myIMAPc.expunge() print "Message deleted" except IMAPClient.Error as err: print "Message deletion failed" print err # Renames a folder def rename_folder(self, oldfolder, newfolder): try: self.myIMAPc.rename_folder(oldfolder, newfolder) print "Folder %s renamed to %s" % (oldfolder, newfolder) except IMAPClient.Error as err: print "Folder renaming failed" print err # Creates a new folder def create_folder(self, folder): try: self.myIMAPc.create_folder(folder) print "New folder %s created" % folder except IMAPClient.Error as err: print "Folder creation failed" print err # Deletes a folder def delete_folder(self, folder): try: self.myIMAPc.delete_folder(folder) print "Folder %s deleted" % folder except IMAPClient.Error as err: print "Folder deletion failed" print err # Creates a new folder and copies the content from the two folders that need to be merged # Then deletes the old folders def merge_folders(self, merged_folder, folder_1, folder_2): try: self.create_folder(merged_folder) # Selects the folder with read/write permission self.myIMAPc.select_folder(folder_1, True) messages = self.myIMAPc.search(['NOT DELETED']) print "Moving %d messages from %s to %s" % (len(messages), folder_1, merged_folder) self.myIMAPc.copy(messages, merged_folder) self.myIMAPc.select_folder(folder_2, True) messages = self.myIMAPc.search(['NOT DELETED']) print "Moving %d messages from %s to %s" % (len(messages), folder_2, merged_folder) self.myIMAPc.copy(messages, merged_folder) print "Deleting %s and %s..." % (folder_1, folder_2) self.delete_folder(folder_1) self.delete_folder(folder_2) print "Merge folder operation succeeded" except IMAPClient.Error as err: print "Merge operation failed" print err def logout(self): self.myIMAPc.logout()