def test_create_folder_if_not_exists_already_exists( logged_in_client: IMAPClient, logger, caplog): assert logged_in_client.folder_exists(EmailFolders.INBOX) create_folder_if_not_exists(logged_in_client, EmailFolders.INBOX, logger) assert logged_in_client.folder_exists(EmailFolders.INBOX) assert "Looks like the folder INBOX already exists." in caplog.text
def test_create_folder_if_not_exists_happy_path(logged_in_client: IMAPClient, logger, teardown, caplog): random_folder = "random-1234567890" assert not logged_in_client.folder_exists(random_folder) create_folder_if_not_exists(logged_in_client, random_folder, logger) teardown.append(lambda client: client.delete_folder(random_folder)) assert logged_in_client.folder_exists(random_folder) assert not caplog.text
def random_folder(logged_in_client: IMAPClient): folder = "random-122122121" if not logged_in_client.folder_exists(folder): logged_in_client.create_folder(folder) yield folder if logged_in_client.folder_exists(folder): logged_in_client.delete_folder(folder) assert not logged_in_client.folder_exists(folder)
def get_email_from_web(self, host, account, password, ssl=True): # pragma: no cover conn = IMAPClient(host, use_uid=True, ssl=ssl) conn.login(account, password) self.assertTrue(conn.folder_exists('Inbox')) select_info = conn.select_folder('Inbox') messages = self.wait_for(lambda: self.has_unseen_emails(conn), timeout=15) response = conn.fetch(messages, ['RFC822']) for msgid, data in response.items(): m = email.message_from_bytes(data[b'RFC822']) email_body = m.get_payload() return email_body
def test_email_shift_not_existing_folder(logged_in_client: IMAPClient, random_mail, logger, teardown, caplog): not_existing_folder = "random-123322" shift_mail( client=logged_in_client, uid=random_mail, source=EmailFolders.INBOX, destination=not_existing_folder, logger=logger, ) assert ( f"Failed email (uid: {random_mail}) to move to {not_existing_folder} folder:" in caplog.text) assert not logged_in_client.folder_exists(not_existing_folder) # check that message exists in the inbox logged_in_client.select_folder(EmailFolders.INBOX, readonly=True) assert random_mail in logged_in_client.search("ALL")
def main(): s_username = raw_input("Source Email: ") s_password = getpass.getpass(prompt="Source Password: "******"Destination Email: ") d_password = getpass.getpass(prompt="Source Password: "******"Run it for real? (yes/*)") destination_folder = 'Migrated Chatlogs' # source server source = IMAPClient('imap.gmail.com', use_uid=True, ssl=True) source.login(s_username, s_password) # destination server destination = IMAPClient('imap.gmail.com', use_uid=True, ssl=True) destination.login(d_username, d_password) select_info = source.select_folder("[Gmail]/Chats", readonly=True) print 'Migrating %s chat messages' % select_info['EXISTS'] print 'Why don\'t you go grab a cup of coffee.. this is going to take a while' chats = source.search(['NOT DELETED']) if not destination.folder_exists(destination_folder): print "creating %s " % destination_folder destination.create_folder(destination_folder) for message in chats: print message fetchData = source.fetch(int(message), ['INTERNALDATE', 'FLAGS', 'RFC822'])[message] if certain == "yes": destination.append(destination_folder, msg=fetchData['RFC822'], flags=fetchData['FLAGS'], msg_time=fetchData['INTERNALDATE']) destination.logout() source.logout()
class SourceDriverMail(SourceDriver): def __init__(self, instanceName: str, settings: Settings, parser: MessageParser) -> None: super().__init__("mail", instanceName, settings, parser) # Settings self.__server = self.getSettingString("server", "") self.__user = self.getSettingString("user", "") self.__password = self.getSettingString("password", "") self.__ssl = self.getSettingBoolean("ssl", True) self.__fix_weak_dh = self.getSettingBoolean("fix_weak_dh", False) self.__allowlist = self.getSettingList("allowlist", []) self.__denylist = self.getSettingList("denylist", []) self.__cleanup = self.getSettingString("cleanup", "") self.__archive_folder = self.getSettingString("archive_folder", "Archive") if self.__cleanup == "Delete": self.print("Cleanup strategy is delete") elif self.__cleanup == "Archive": self.print("Cleanup strategy is archive") elif self.__cleanup == "": self.print("Cleanup is disabled") else: self.fatal("Unknown cleanup strategy") # Internal self.__healthy = False self.__connect() def retrieveEvent(self) -> Optional[SourceEvent]: try: if self.isDebug(): self.print("Checking for new mails") messages = self.__imap_client.search('UNSEEN') for uid, message_data in self.__imap_client.fetch( messages, "RFC822").items(): message = email.message_from_bytes(message_data[b"RFC822"], policy=policy.default) sender = parseaddr(message.get("From"))[1] sourceEvent = SourceEvent() sourceEvent.source = SourceEvent.SOURCE_MAIL sourceEvent.timestamp = datetime.datetime.strptime( message.get('Date'), "%a, %d %b %Y %H:%M:%S %z").strftime( SourceEvent.TIMESTAMP_FORMAT) sourceEvent.sender = sender sourceEvent.raw = message.get_body(('plain', )).get_content() if self.isSenderAllowed(allowlist=self.__allowlist, denylist=self.__denylist, sender=sender): parsedSourceEvent = self.parser.parseMessage( sourceEvent, None) # type: ignore[union-attr] self.__do_cleanup(uid) return parsedSourceEvent else: self.error("Received unhandled message (ignored sender)") return UnhandledEvent.fromSourceEvent( sourceEvent, UnhandledEvent.CAUSE_IGNORED_SENDER) except (timeout, OSError) as e: self.error("Connection to mailserver timed out") self.__healthy = False self.__connect() def getSourceState(self) -> SourceState: if self.__healthy: return SourceState.OK return SourceState.ERROR def __connect(self): if self.__fix_weak_dh: context = ssl.SSLContext( ssl.PROTOCOL_TLSv1_2) # Workaround for weak dh key context.set_ciphers('DEFAULT@SECLEVEL=1') else: context = ssl.SSLContext() try: if self.isDebug(): self.print("Connecting to server {}".format(self.__server)) self.__imap_client = IMAPClient(self.__server, use_uid=True, ssl=self.__ssl, ssl_context=context, timeout=1.0) except gaierror: self.error("Failed to connect to Mail Server") except ssl.SSLError: self.fatal("Failed to connect to Mail Server (TLS Error)") else: try: if self.isDebug(): self.print("Login as user {}".format(self.__user)) self.__imap_client.login(self.__user, self.__password) except LoginError: self.error("Mail Server login failed") else: if self.isDebug(): self.print("Login successful") self.__healthy = True self.__imap_client.select_folder('INBOX', readonly=False) def __create_imap_folder(self, folder): if not self.__imap_client.folder_exists(folder): self.print("Folder {} does not exist creating") self.__imap_client.create_folder(folder) def __do_cleanup(self, uid): if self.__cleanup == "Archive": self.__create_imap_folder(self.__archive_folder) self.__imap_client.copy(uid, self.__archive_folder) if self.__cleanup == "Delete" or self.__cleanup == "Archive": self.__imap_client.delete_messages(uid) self.__imap_client.expunge(uid)
from imapclient import IMAPClient HOST = 'imap.gmail.com' USERNAME = '******' PASSWORD = '******' ALL_MAIL = '[Gmail]/All Mail' SSL = True try: server = IMAPClient(HOST, use_uid=True, ssl=SSL) except: raise Exception('Could not successfully connect to the IMAP host') server.login(USERNAME, PASSWORD) if not server.folder_exists(ALL_MAIL): for folder in server.xlist_folders(): labels = folder[0] if 'AllMail' in labels[-1]: ALL_MAIL = folder[2] server.select_folder(ALL_MAIL) #print "Available folders:" #for f in server.list_folders(): # print f[-1] #raise Exception('Your "All Mail" label is either not visible or in another language!') # that's why we only support gmail # for other mail services we'd have to translate the custom # search to actual IMAP queries, thus no X-GM-RAW cookie to us
class ImapConfiguration(DataRetriever): def __init__(self, instrument_id, logger, configuration=None): super().__init__(instrument_id, logger) if configuration is None: self.configuration["Server"] = None self.configuration["Port"] = None self.configuration["User"] = None self.configuration["Password"] = None self.configuration["Source Folder"] = None self.configuration["Downloaded Folder"] = None else: self.configuration = configuration # For storing the current IMAP connection self.imapconn = None # For storing the current message ID during processing self.current_id = None @staticmethod def get_type(): return "IMAP Email" # Check that the configuration works def test_configuration(self): config_ok = True logged_in = False # Connect try: self.imapconn = IMAPClient(host=self.configuration["Server"], port=self.configuration["Port"]) except: self.log(logging.CRITICAL, "Cannot connect to IMAP server: " + traceback.format_exc()) config_ok = False # Authenticate try: if self.imapconn is not None: self.imapconn.login(self.configuration["User"], self.configuration["Password"]) logged_in = True # Check folders if not self.imapconn.folder_exists(self.configuration["Source Folder"]): self.log(logging.CRITICAL, "Source Folder does not exist") config_ok = False if not self.imapconn.folder_exists(self.configuration["Downloaded Folder"]): self.log(logging.CRITICAL, "Downloaded Folder does not exist") config_ok = False except: self.log(logging.CRITICAL, "Cannot log in to IMAP server: " + traceback.format_exc()) config_ok = False # Shut everything down self.shutdown() return config_ok # Initialise a connection to the IMAP server def startup(self): result = True try: # Log in to the mail server self.imapconn = IMAPClient(host=self.configuration["Server"], port=self.configuration["Port"]) self.imapconn.login(self.configuration["User"], self.configuration["Password"]) # Get the list of messages we can process self.imapconn.select_folder(self.configuration["Source Folder"]) self.message_ids = self.imapconn.search(["NOT", "DELETED"]) self.current_index = -1 except: self.log(logging.CRITICAL, "Cannot log in to IMAP server: " + traceback.format_exc()) result = False return result # Shutdown the server connection def shutdown(self): if self.imapconn is not None: try: if logged_in: self.imapconn.logout() self.imapconn.shutdown() except: # Don't care pass self.imapconn = None # Get the next message and extract its attachment def _retrieve_next_file(self): try: file_found = False self.current_index = self.current_index + 1 while not file_found and self.current_index < len(self.message_ids): self.log(logging.DEBUG, "Processing email ID " + \ str(self.message_ids[self.current_index])) message_content = self.imapconn.fetch( \ self.message_ids[self.current_index], "RFC822") \ [self.message_ids[self.current_index]][b"RFC822"] message = email.message_from_bytes(message_content) for part in message.walk(): content_disposition = part.get("Content-Disposition") if content_disposition is not None and \ content_disposition.startswith("attachment"): filename = self._extract_filename(part.get_filename()) self.log(logging.DEBUG, "Extracting attachment " + filename) contents = part.get_payload(decode=True) self._add_file(filename, contents) file_found = True if not file_found: self.imapconn.move(self.message_ids[self.current_index], self.configuration["Downloaded Folder"]) self.current_index = self.current_index + 1 except: self.log(logging.ERROR, "Failed to retrieve next file: " + traceback.format_exc()) # Extract a filename, handling encoded filenames as required def _extract_filename(self, filename): result = filename if decode_header(filename)[0][1] is not None: result = decode_header(filename)[0][0].decode(decode_header(filename)[0][1]) return result # The file(s) have been processed successfully; # clean them up accordingly def _cleanup_success(self): try: self.log(logging.DEBUG, "Moving email " + str(self.message_ids[self.current_index]) + " to downloaded folder") self.imapconn.move(self.message_ids[self.current_index], self.configuration["Downloaded Folder"]) except: self.log(logging.ERROR, "Failed to move email after processing: " + traceback.format_exc()) # The file(s) were not processed successfully; # clean them up accordingly def _cleanup_fail(self): # We don't do anything - we'll try to process # the mail again next time round pass # The file(s) were not processed def _cleanup_not_processed(self): # No action required pass
class Server: """ Server class to fetch and filter data Connects to the IMAP server, search according to a criteria, fetch all attachments of the mails matching the criteria and save them locally with a timestamp """ def __init__(self, host, username, password, debug=False): """ Server class __init__ which expects an IMAP host to connect to @param host: gmail's default server is fine: imap.gmail.com @param username: your gmail account (i.e. [email protected]) @param password: we highly recommend you to use 2-factor auth here """ if not host: raise Exception('Missing IMAP host parameter in your config') try: self._server = IMAPClient(host, use_uid=True, ssl=True) except: raise Exception('Could not successfully connect to the IMAP host') setattr(self._server, 'debug', debug) # mails index to avoid unnecessary redownloading index = '.index_%s' % (username) index = os.path.join(_app_folder(), index) self._index = shelve.open(index, writeback=True) # list of attachments hashes to avoid dupes hashes = '.hashes_%s' % (username) hashes = os.path.join(_app_folder(), hashes) self._hashes = shelve.open(hashes, writeback=True) self._username = username self._login(username, password) def _login(self, username, password): """ Login to the IMAP server and selects the all mail folder @param username: your gmail account (i.e. [email protected]) @param password: we highly recommend you to use 2-factor auth here """ if not username or not password: raise Exception('Missing username or password parameters') # you may want to hack this to only fetch attachments from a # different exclusive label (assuming you have them in english) all_mail = '[Gmail]/All Mail' try: self._server.login(username, password) except: raise Exception('Cannot login, check username/password, are you using 2-factor auth?') # this is how we get the all mail folder even if the user's # gmail interface is in another language, not in english if not self._server.folder_exists(all_mail): for folder in self._server.xlist_folders(): labels = folder[0] if 'AllMail' in labels[-1]: all_mail = folder[2] self._server.select_folder(all_mail) def _filter_messages(self): """Filter mail to only parse ones containing images""" # creates a list of all types of image files to search for, # even though we have no idea if gmail supports them or what mimetypes.init() mimes = [] for ext in mimetypes.types_map: if 'image' in mimetypes.types_map[ext]: mimes.append(ext.replace('.', '')) mimelist = ' OR '.join(mimes) # that's why we only support gmail # for other mail services we'd have to translate the custom # search to actual IMAP queries, thus no X-GM-RAW cookie for us criteria = 'X-GM-RAW "has:attachment filename:(%s)"' % (mimelist) try: messages = self._server.search([criteria]) except: raise Exception('Search criteria return a failure, it must be a valid gmail search') # stats logging print 'LOG: %d messages matched the search criteria %s' % (len(messages), criteria) return messages def lostphotosfound(self): """The actual program, which fetchs the mails and all its parts attachments""" messages = self._filter_messages() for msg in messages: try: idfetched = self._server.fetch([msg], ['X-GM-MSGID']) except: raise Exception('Could not fetch the message ID, server did not respond') msgid = str(idfetched[idfetched.keys()[0]]['X-GM-MSGID']) # mail has been processed in the past, skip it if msgid in self._index.keys(): print 'Skipping X-GM-MSDID %s' % (msgid) continue # if it hasn't, fetch it and iterate through its parts msgdata = self._server.fetch([msg], ['RFC822']) for data in msgdata: mail = message_from_string(msgdata[data]['RFC822'].encode('utf-8')) if mail.get_content_maintype() != 'multipart': continue # logging header_from = _charset_decoder(mail['From']) header_subject = _charset_decoder(mail['Subject']) print '[%s]: %s' % (header_from, header_subject) for part in mail.walk(): # if it's only plain text, i.e. no images if part.get_content_maintype() == 'multipart': continue # if no explicit attachments unless they're inline if part.get('Content-Disposition') is None: pass # if non-graphic inline data if not 'image/' in part.get_content_type(): continue # only then we can save this mail part self._save_part(part, mail) # all parts of mail processed, add it to the index self._index[msgid] = msgid def _save_part(self, part, mail): """ Internal function to decode attachment filenames and save them all @param mail: the mail object from message_from_string so it can checks its date @param part: the part object after a mail.walk() to get multiple attachments """ if not hasattr(self, "seq"): self.seq = 0 # we check if None in filename instead of just if it is None # due to the type of data decode_header returns to us header_filename = _charset_decoder(part.get_filename()) # i.e. some inline attachments have no filename field in the header # so we have to hack around it and get the name field if 'None' in header_filename: header_filename = part.get('Content-Type').split('name=')[-1].replace('"', '') elif not header_filename[0][0] or header_filename[0][0] is None: # we should hopefully never reach this, attachments would be 'noname' in gmail header_filename = 'attachment-%06d.data' % (self.seq) self.seq += 1 # sanitize it punct = '!"#$&\'*+/;<>?[\]^`{|}~' header_filename = header_filename.translate(None, punct) # 2012-10-28_19-15-22 (Y-M-D_H-M-S) header_date = parsedate(mail['date']) header_date = '%s-%s-%s_%s-%s-%s_' % (header_date[0], header_date[1], header_date[2], header_date[3], header_date[4], header_date[5]) filename = header_date + header_filename # we should create it in the documents folder username = self._username userdir = os.path.expanduser('~/LostPhotosFound') savepath = os.path.join(userdir, username) if not os.path.isdir(savepath): os.makedirs(savepath) # logging complement print '\t...%s' % (filename) saved = os.path.join(savepath, filename) if not os.path.isfile(saved): with open(saved, 'wb') as imagefile: try: payload = part.get_payload(decode=True) except: message = 'Failed when downloading attachment: %s' % (saved) raise Exception(message) payload_hash = hashlib.sha1(payload).hexdigest() # gmail loves to duplicate attachments in replies if payload_hash not in self._hashes.keys(): try: imagefile.write(payload) except: message = 'Failed writing attachment to file: %s' % (saved) raise Exception(message) self._hashes[payload_hash] = payload_hash else: print 'Duplicated attachment %s (%s)' % (saved, payload_hash) os.remove(saved) def close(self): """Gracefully sync/close indexes and disconnects from the IMAP server""" self._index.sync() self._index.close() self._hashes.sync() self._hashes.close() self._server.close_folder() self._server.logout()
class Hauberk(): def __init__(self, dry_run=False): self.rules = list() self.dry = dry_run def login(self, username, password, server): logger.info("Connecting to Server") self.client = IMAPClient(server, use_uid=True) self.client.login(username, password) logger.info("Connected") def add_rule(self, filters, actions): self.rules.append(Rule(filters=filters, actions=actions)) if not filters: raise Exception("Must define filters in rule") if not actions: raise Exception("Must define actions in rule") logger.info("Added rule number %s", len(self.rules)) def run(self): messages = self.client.search() logger.info("%s messages to process", len(messages)) for msgid, data in self.client.fetch( messages, [FetchFlags.ENVELOPE.value, FetchFlags.BODY.value]).items(): envelope = data[FetchFlags.ENVELOPE.value] body = data[FetchFlags.BODY.value] message = Message(envelope, body) logger.debug("Processing message %s: %s", msgid, message.subject) done = False for rule in self.rules: for _filter in rule.filters: logger.debug("Running filter %s", _filter) if _filter(message): logger.debug("Match found!") # log here that there was a match for action in rule.actions: logger.debug("Running action %s", action) action(self.client, msgid, self.dry) done = True break if done: break def select_folder(self, folder): """Select working folder. :param folder: Folder rules should be applied to """ logger.info("Running in folder %s", folder) self.client.select_folder(folder) def existing_folders(self, folders): """Make sure certain folders exist client side. :param folders: List of folder names """ logger.info("Checking for existing folders") if not isinstance(folders, list): folders = [folders] for folder in folders: logger.debug("Checking for existance of folder %s" % folder) if not self.client.folder_exists(folder): logger.warning("Folder %s does not exist, creating" % folder) if not self.dry: self.client.create_folder(folder)
class IMAP(): """ Central class for IMAP server communication """ Retval = namedtuple('Retval', 'code data') def __init__(self, logger, username, password, server='localhost', port=143, starttls=False, imaps=False, tlsverify=True, test=False, timeout=None): self.logger = logger self.username = username self.password = password self.server = server self.port = port self.imaps = imaps self.starttls = starttls self.timeout = timeout self.sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) # TODO add proto arg if tlsverify: self.sslcontext.verify_mode = ssl.CERT_REQUIRED else: self.sslcontext.verify_mode = ssl.CERT_NONE # TODO improve? self.test = test self.conn = None def do_select_mailbox(func): """ Decorator to do a fresh mailbox SELECT """ def wrapper(*args, **kwargs): if len(args) != 1: raise AttributeError( 'Size of *args tuple "{0}" isn\'t 1. It looks like you haven\'t specified all ' 'method arguments as named arguments!'.format( args)) mailbox = None for key in ['mailbox', 'source']: if key in kwargs.keys(): mailbox = kwargs[key] break if mailbox is None: raise KeyError('Unable to SELECT a mailbox, kwargs "{0}" doesn\'t contain a mailbox name'.format(kwargs)) result = args[0].select_mailbox(mailbox) if not result.code: raise RuntimeError(result.data) return func(*args, **kwargs) return wrapper def process_error(self, exception, simple_return=False): """ Process Python exception by logging a message and optionally showing traceback """ trace_info = exc_info() err_msg = str(exception) if isinstance(exception, IMAPClient.Error): err_msg = Helper().byte_to_str(exception.args[0]) self.logger.error('Catching IMAP exception %s: %s', type(exception), err_msg) if self.logger.isEnabledFor(loglevel_DEBUG): print_exception(*trace_info) del trace_info if simple_return: return exception else: return self.Retval(False, err_msg) def connect(self, retry=True, logout=False): """ Connect to IMAP server and login """ if self.starttls: self.logger.debug('Establishing IMAP connection using STARTTLS/%s to %s and logging in with user %s', self.port, self.server, self.username) elif self.imaps: self.logger.debug('Establishing IMAP connection using SSL/%s (imaps) to %s and logging in with user %s', self.port, self.server, self.username) login = '' err_return = None try: self.conn = IMAPClient(host=self.server, port=self.port, use_uid=True, ssl=self.imaps, ssl_context=self.sslcontext, timeout=self.timeout) if self.starttls: self.conn.starttls(ssl_context=self.sslcontext) login = self.conn.login(self.username, self.password) login_response = Helper().byte_to_str(login) # Test login/auth status login_success = False noop = self.noop() if noop.code and noop.data: login_success = True if logout: return self.disconnect() elif login_success: return self.Retval(True, login_response) else: return self.Retval(False, login_response) # pragma: no cover except Exception as e: err_return = self.process_error(e) if err_return.data == '[AUTHENTICATIONFAILED] Authentication failed.': return err_return if retry: self.logger.error('Trying one more time to login') sleep(2) return self.connect(retry=False, logout=logout) return err_return def noop(self): """ Do a noop to test login status """ try: noop = self.conn.noop() noop_response = Helper().byte_to_str(noop[0]) noop_resp_pattern_re = regex_compile('^(Success|NOOP completed)') login_success = noop_resp_pattern_re.match(noop_response) return self.Retval(True, login_success) except IMAPClient.Error as e: return self.process_error(e) def disconnect(self): """ Disconnect from IMAP server """ result = self.conn.logout() response = Helper().byte_to_str(result) return self.Retval(response == 'Logging out', response) def list_mailboxes(self, directory='', pattern='*'): """ Get a listing of folders (mailboxes) on the server """ try: raw_list = self.conn.list_folders(directory, pattern) nice_list = [] for mailbox in raw_list: flags = [] for flag in mailbox[0]: flags.append(flag.decode('utf-8')) nice_list.append({'name': mailbox[2], 'flags': flags, 'delimiter': mailbox[1].decode("utf-8")}) return self.Retval(True, nice_list) except IMAPClient.Error as e: return self.process_error(e) def select_mailbox(self, mailbox): """ Select a mailbox to work on """ self.logger.debug('Switching to mailbox %s', mailbox) try: result = self.conn.select_folder(mailbox) response = {} for key, value in result.items(): unicode_key = Helper().byte_to_str(key) if unicode_key == 'FLAGS': flags = [] for flag in value: flags.append(Helper().byte_to_str(flag)) response[unicode_key] = tuple(flags) else: response[unicode_key] = value return self.Retval(True, response) except IMAPClient.Error as e: return self.process_error(e) def add_mail(self, mailbox, message, flags=(), msg_time=None): """ Add/append a mail to a mailbox """ self.logger.debug('Adding a mail into mailbox %s', mailbox) try: if not isinstance(message, Mail): message = Mail(logger=self.logger, mail_native=message) message_native = message.get_native() #self.conn.append(mailbox, message, flags, msg_time) self._append(mailbox, str(message_native), flags, msg_time) # According to rfc4315 we must not return the UID from the response, so we are fetching it ourselves uids = self.search_mails(mailbox=mailbox, criteria='HEADER Message-Id "{0}"'.format(message.get_header('Message-Id'))).data[0] return self.Retval(True, uids) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def search_mails(self, mailbox, criteria='ALL', autocreate_mailbox=False): """ Search for mails in a mailbox """ self.logger.debug('Searching for mails in mailbox %s and criteria=\'%s\'', mailbox, criteria) try: return self.Retval(True, list(self.conn.search(criteria=criteria))) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def fetch_mails(self, uids, mailbox, return_fields=None): """ Retrieve mails from a mailbox """ self.logger.debug('Fetching mails with uids %s', uids) return_raw = True if return_fields is None: return_raw = False return_fields = [b'RFC822'] mails = {} try: for uid in uids: result = self.conn.fetch(uid, return_fields) if not result: continue if return_raw: mails[uid] = result[uid] else: #mails[uid] = Mail(logger=self.logger, uid=uid, mail_native=email.message_from_bytes(result[uid][b'RFC822'])) mails[uid] = Mail(logger=self.logger, mail_native=email.message_from_bytes(result[uid][b'RFC822'])) return self.Retval(True, mails) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def get_mailflags(self, uids, mailbox): """ Retrieve flags from mails """ try: result = self.conn.get_flags(uids) flags = {} for uid in uids: flags[uid] = [] if uid not in result.keys(): self.logger.error('Failed to get flags for mail with uid=%s: %s', uid, result) return self.Retval(False, None) for flag in result[uid]: flags[uid].append(flag.decode('utf-8')) return self.Retval(True, flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def set_mailflags(self, uids, mailbox, flags=[]): """ Set and retrieve flags from mails """ if self.test: self.logger.info('Would have set mail flags on message uids "%s"', str(uids)) return self.Retval(True, None) else: self.logger.debug('Setting flags=%s on mails uid=%s', flags, uids) try: result = self.conn.set_flags(uids, flags) _flags = {} for uid in uids: _flags[uid] = [] if uid not in result.keys(): self.logger.error('Failed to set and get flags for mail with uid=%s: %s', uid, result) return self.Retval(False, None) for flag in result[uid]: _flags[uid].append(flag.decode('utf-8')) return self.Retval(True, _flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def add_mailflags(self, uids, mailbox, flags=[]): """ Add and retrieve flags from mails """ if self.test: self.logger.info('Would have added mail flags on message uids "%s"', str(uids)) return self.Retval(True, None) else: self.logger.debug('Adding flags=%s on mails uid=%s', flags, uids) try: result = self.conn.add_flags(uids, flags) _flags = {} for uid in uids: _flags[uid] = [] if uid not in result.keys(): self.logger.error('Failed to add and get flags for mail with uid=%s: %s', uid, result) return self.Retval(False, None) for flag in result[uid]: _flags[uid].append(flag.decode('utf-8')) return self.Retval(True, _flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def move_mail(self, message_ids, source, destination, delete_old=True, expunge=True, add_flags=None, set_flags=None): """ Move a mail from a mailbox to another """ return self.copy_mails(message_ids=message_ids, source=source, destination=destination, delete_old=delete_old, expunge=expunge, add_flags=add_flags, set_flags=set_flags) @do_select_mailbox def copy_mails(self, source, destination, message_ids=None, delete_old=False, expunge=False, add_flags=None, set_flags=None): """ Copies one or more mails from a mailbox into another """ if self.test: if delete_old: self.logger.info('Would have moved mail Message-Ids="%s" from "%s" to "%s", skipping because of beeing in testmode', message_ids, source, destination) else: self.logger.info('Would have copied mails with Message-Ids="%s" from "%s" to "%s", skipping because of beeing in testmode', message_ids, source, destination) return self.Retval(True, None) else: try: if delete_old: self.logger.debug('Moving mail Message-Ids="%s" from "%s" to "%s"', message_ids, source, destination) else: self.logger.debug('Copying mail Message-Ids="%s" from "%s" to "%s"', message_ids, source, destination) #if message_ids is None: # message_ids = [] # result = self.fetch_mails(uids=uids, mailbox=source) # if not result.code: # self.logger.error('Failed to determine Message-Id by uids for mail with uids "%s"', uids) # return result # message_ids.append(result.data.keys()) if not self.mailbox_exists(destination).data: self.logger.info('Destination mailbox %s doesn\'t exist, creating it for you', destination) result = self.create_mailbox(mailbox=destination) if not result.code: self.logger.error('Failed to create the mailbox %s: %s', source, result.data) # pragma: no cover return result # pragma: no cover uids = [] for message_id in message_ids: result = self.search_mails(mailbox=source, criteria='HEADER Message-Id "{0}"'.format(message_id)) if not result.code or len(result.data) == 0: self.logger.error('Failed to determine uid by Message-Id for mail with Message-Id "%s"', message_id) return self.Retval(False, result.data) uids.append(result.data[0]) result = self.select_mailbox(source) if not result.code: return result # pragma: no cover self.conn.copy(uids, destination) if delete_old: result = self.delete_mails(uids=uids, mailbox=source) if not result.code: self.logger.error('Failed to remove old mail with Message-Id="%s"/uids="%s": %s', message_ids, uids, result.data) # pragma: no cover return result # pragma: no cover if expunge: # TODO don't expunge by default result = self.expunge(mailbox=source) if not result.code: self.logger.error('Failed to expunge on mailbox %s: %s', source, result.data) # pragma: no cover return result # pragma: no cover dest_uids = [] for message_id in message_ids: result = self.search_mails(mailbox=destination, criteria='HEADER Message-Id "{0}"'.format(message_id)) if not result.code: self.logger.error('Failed to determine uid by Message-Id for mail with Message-Id "%s"', message_id) # pragma: no cover return result # pragma: no cover dest_uids.append(result.data[0]) if isinstance(set_flags, list): self.set_mailflags(uids=dest_uids, mailbox=destination, flags=set_flags) if add_flags: self.add_mailflags(uids=dest_uids, mailbox=destination, flags=add_flags) return self.Retval(True, dest_uids) except IMAPClient.Error as e: return self.process_error(e) def _append(self, folder, msg, flags=(), msg_time=None): # TODO """ FORKED FORM IMAPCLIENT """ if msg_time: if not msg_time.tzinfo: # pragma: no cover msg_time = msg_time.replace(tzinfo=FixedOffset.for_system()) # pragma: no cover time_val = '"{0}"'.format(msg_time.strftime("%d-%b-%Y %H:%M:%S %z")) time_val = imapclient.imapclient.to_unicode(time_val) else: time_val = None return self.conn._command_and_check('append', self.conn._normalise_folder(folder), imapclient.imapclient.seq_to_parenstr(flags), time_val, Helper.str_to_bytes(msg), unpack=True) @do_select_mailbox def expunge(self, mailbox): """ Expunge mails form a mailbox """ self.logger.debug('Expunge mails from mailbox %s', mailbox) try: return self.Retval(True, b'Expunge completed.' in self.conn.expunge()) except IMAPClient.Error as e: # pragma: no cover return self.process_error(e) # pragma: no cover def create_mailbox(self, mailbox): """ Create a mailbox """ self.logger.debug('Creating mailbox %s', mailbox) try: return self.Retval(True, self.conn.create_folder(mailbox) == b'Create completed.') except IMAPClient.Error as e: return self.process_error(e) def mailbox_exists(self, mailbox): """ Check whether a mailbox exists """ try: return self.Retval(True, self.conn.folder_exists(mailbox)) except IMAPClient.Error as e: # pragma: no cover return self.process_error(e) # pragma: no cover @do_select_mailbox def delete_mails(self, uids, mailbox): """ Delete mails """ self.logger.debug('Deleting mails with uid="%s"', uids) try: result = self.conn.delete_messages(uids) flags = {} for uid in uids: flags[uid] = [] if uid not in result.keys(): self.logger.error('Failed to get flags for mail with uid=%s after deleting it: %s', uid, result) return self.Retval(False, None) for flag in result[uid]: flags[uid].append(flag.decode('utf-8')) return self.Retval(True, flags) except IMAPClient.Error as e: return self.process_error(e)
class IMAP(Thread): def __init__(self, filman): super(IMAP, self).__init__() self.imap = IMAPClient(HOST, use_uid=True, ssl=ssl) self.imap.login(USERNAME, PASSWORD) self.messages = [] self.filterman = filman self.counter = 0 self.check_dests() self.loop() def check_dests(self): dests = self.filterman.get_dests() for d in dests: if not self.imap.folder_exists(d): self.imap.create_folder(d) logging.info('[create folder] %s' % d) if not self.imap.folder_exists(default_not_matched_dest): self.imap.create_folder(default_not_matched_dest) logging.info('[create folder] %s' % default_not_matched_dest) def mark_as_unread(self, msgs): return self.imap.remove_flags(msgs, ('\\SEEN')) def check(self): server = self.imap select_info = server.select_folder('INBOX') logging.info("source imap inited: %r" % select_info) messages = server.search(['NOT SEEN']) messages = sorted(messages, reverse=True) self.messages = list(messages) logging.info('got %d unread messages' % len(self.messages)) def idle(self, secs=30): server = self.imap server.idle() responses = server.idle_check(timeout=secs) text, responses = server.idle_done() logging.info('idle response: %s' % (responses)) return not responses def loop(self): logging.info('enter loop %d' % self.counter) self.counter += 1 self.check() while self.messages: self._dozen() self.imap.close_folder() def _dozen(self): if self.messages: msgs = self.messages[:12] self.messages = self.messages[12:] else: return logging.info('processing the first %d msgs; left %d...' % ( len(msgs), len(self.messages))) logging.info(msgs) response = self.imap.fetch(msgs, ['RFC822']) msgs = [(msgid, Msg(string=data['RFC822'])) for (msgid, data) in response.iteritems()] self.filterman.test_match_and_take_action(self.imap, msgs) def run(self): count = 0 while True: count += 1 logging.info('idle counter: %d' % count) self.idle() or self.loop() sleep(10) if not count % 5: # do loop every 10 runs. self.loop()
class IMAPBot(object): IMAPBotError = IMAPBotError def __init__(self, host, username, password, ssl=True): self.server = IMAPClient(host, use_uid=True, ssl=ssl) self.server.login(username, password) if 'IDLE' not in self.server.capabilities(): raise IMAPBotError('Sorry, this IMAP server does not support IDLE.') # the folder where processed emails go self.processed_folder = 'imapbot_processed' self.idle_timeout = 5 # seconds self._is_idle = False self._run = True self._create_folder(self.processed_folder) def check_mail(self): select_info = self.server.select_folder('INBOX') print '%d messages in INBOX' % select_info['EXISTS'] messages = self.server.search(['UNSEEN']) messages = self.server.search(['NOT DELETED']) print "%d messages that haven't been seen" % len(messages) if not messages: return #response = self.server.fetch(messages, ['FLAGS', 'INTERNALDATE', 'RFC822.SIZE', 'ENVELOPE', 'RFC822.TEXT']) response = self.server.fetch(messages, ['FLAGS', 'ENVELOPE', 'RFC822.TEXT']) for message_id, data in response.iteritems(): message = Message(message_id, data['ENVELOPE'], data['RFC822.TEXT'], data['FLAGS']) self.process(message) def complete(self, message): message_ids = [message.id] self.server.copy(message_ids, self.processed_folder) self.server.delete_messages(message_ids) self.server.expunge() def _create_folder(self, name): # make sure the folder doesn't already exist if self.server.folder_exists(name): return self.server.create_folder(name) def handle_message(self, message): print 'message id: {}, from: {}:'.format(message.id, message.envelope.get_email('from')) with open('message.txt', 'ab') as fh: fh.write('{}\n\n'.format(message.text)) print message.plain or message.html or 'no message' def idle(self): if self._is_idle: return self.server.idle() self._is_idle = True return True # this actually changed state def unidle(self): if not self._is_idle: return self.server.idle_done() self._is_idle = False return True # this call actually changed state def process(self, message): self.handle_message(message) self.complete(message) def run(self): # process any mail that was in the inbox before coming online self.check_mail() # put the connection in idle mode so we get notifications self.idle() # loop forever looking for stuff while self._run: for message in self.server.idle_check(timeout=self.idle_timeout): if message[0] == 'OK': continue with Unidle(self): self.check_mail() def quit(self): self._run = False self.unidle() print self.server.logout()
class IMAP(): """ Central class for IMAP server communication """ Retval = namedtuple('Retval', 'code data') def __init__(self, logger, username, password, server='localhost', port=143, starttls=False, imaps=False, tlsverify=True, test=False, timeout=None): self.logger = logger self.username = username self.password = password self.server = server self.port = port self.imaps = imaps self.starttls = starttls self.timeout = timeout self.sslcontext = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) if tlsverify: self.sslcontext.verify_mode = ssl.CERT_REQUIRED else: self.sslcontext.check_hostname = False self.sslcontext.verify_mode = ssl.CERT_NONE self.test = test self.conn = None def do_select_mailbox(func): """ Decorator to do a fresh mailbox SELECT """ def wrapper(*args, **kwargs): if len(args) != 1: raise AttributeError( 'Size of *args tuple "{0}" isn\'t 1. It looks like you haven\'t specified all ' 'method arguments as named arguments!'.format(args)) mailbox = None for key in ['mailbox', 'source']: if key in kwargs.keys(): mailbox = kwargs[key] break if mailbox is None: raise KeyError( 'Unable to SELECT a mailbox, kwargs "{0}" doesn\'t contain a mailbox name' .format(kwargs)) result = args[0].select_mailbox(mailbox) if not result.code: raise RuntimeError(result.data) return func(*args, **kwargs) return wrapper def process_error(self, exception, simple_return=False): """ Process Python exception by logging a message and optionally showing traceback """ trace_info = exc_info() err_msg = str(exception) if isinstance(exception, IMAPClient.Error): err_msg = Helper().byte_to_str(exception.args[0]) self.logger.error("Catching IMAP exception {}: {}".format( type(exception), err_msg)) if self.logger.isEnabledFor(loglevel_DEBUG): print_exception(*trace_info) del trace_info if simple_return: return exception else: return self.Retval(False, err_msg) def connect(self, retry=True, logout=False): """ Connect to IMAP server and login """ if self.starttls: self.logger.debug( 'Establishing IMAP connection using STARTTLS/{} to {} and logging in with user {}' .format(self.port, self.server, self.username)) elif self.imaps: self.logger.debug( 'Establishing IMAP connection using SSL/{} (imaps) to {} and logging in with user {}' .format(self.port, self.server, self.username)) try: self.conn = IMAPClient(host=self.server, port=self.port, use_uid=True, ssl=self.imaps, ssl_context=self.sslcontext, timeout=self.timeout) if self.starttls: self.conn.starttls(ssl_context=self.sslcontext) login = self.conn.login(self.username, self.password) login_response = Helper().byte_to_str(login) # Test login/auth status login_success = False noop = self.noop() if noop.code and noop.data: login_success = True if logout: return self.disconnect() elif login_success: return self.Retval(True, login_response) else: return self.Retval(False, login_response) # pragma: no cover except exceptions.LoginError as e: return self.process_error(e) except Exception as e: err_return = self.process_error(e) if retry: self.logger.error('Trying one more time to login') sleep(2) return self.connect(retry=False, logout=logout) return err_return def noop(self): """ Do a noop to test login status """ try: noop = self.conn.noop() noop_response = Helper().byte_to_str(noop[0]) noop_resp_pattern_re = regex_compile('^(Success|NOOP completed)') login_success = noop_resp_pattern_re.match(noop_response) return self.Retval(True, login_success) except IMAPClient.Error as e: return self.process_error(e) def disconnect(self): """ Disconnect from IMAP server """ result = self.conn.logout() response = Helper().byte_to_str(result) return self.Retval(response == 'Logging out', response) def list_mailboxes(self, directory='', pattern='*'): """ Get a listing of folders (mailboxes) on the server """ try: raw_list = self.conn.list_folders(directory, pattern) nice_list = [] for mailbox in raw_list: flags = [] for flag in mailbox[0]: flags.append(flag.decode('utf-8')) nice_list.append({ 'name': mailbox[2], 'flags': flags, 'delimiter': mailbox[1].decode("utf-8") }) return self.Retval(True, nice_list) except IMAPClient.Error as e: return self.process_error(e) def select_mailbox(self, mailbox): """ Select a mailbox to work on """ self.logger.debug('Switching to mailbox {}'.format(mailbox)) try: result = self.conn.select_folder(mailbox) response = {} for key, value in result.items(): unicode_key = Helper().byte_to_str(key) if unicode_key == 'FLAGS': flags = [] for flag in value: flags.append(Helper().byte_to_str(flag)) response[unicode_key] = tuple(flags) else: response[unicode_key] = value return self.Retval(True, response) except IMAPClient.Error as e: return self.process_error(e) def add_mail(self, mailbox, message, flags=(), msg_time=None): """ Add/append a mail to a mailbox """ self.logger.debug('Adding a mail into mailbox {}'.format(mailbox)) try: if not isinstance(message, Mail): message = Mail(logger=self.logger, mail_native=message) self.conn.append(mailbox, str(message.get_native()), flags, msg_time) # According to rfc4315 we must not return the UID from the response, so we are fetching it ourselves uids = self.search_mails(mailbox=mailbox, criteria='HEADER Message-Id "{}"'.format( message.get_message_id())).data[0] return self.Retval(True, uids) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def search_mails(self, mailbox, criteria='ALL', autocreate_mailbox=False): """ Search for mails in a mailbox """ self.logger.debug( 'Searching for mails in mailbox {} and criteria=\'{}\''.format( mailbox, criteria)) try: return self.Retval(True, list(self.conn.search(criteria=criteria))) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def fetch_mails(self, uids, mailbox, return_fields=None): """ Retrieve mails from a mailbox """ self.logger.debug('Fetching mails with uids {}'.format(uids)) return_raw = True if return_fields is None: return_raw = False return_fields = [b'RFC822'] mails = {} try: for uid in uids: result = self.conn.fetch(uid, return_fields) if not result: continue if return_raw: mails[uid] = result[uid] else: # mails[uid] = Mail(logger=self.logger, uid=uid, mail_native=email.message_from_bytes(result[uid][b'RFC822'])) mails[uid] = Mail(logger=self.logger, mail_native=email.message_from_bytes( result[uid][b'RFC822'])) return self.Retval(True, mails) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def get_mailflags(self, uids, mailbox): """ Retrieve flags from mails """ try: result = self.conn.get_flags(uids) flags = {} for uid in uids: flags[uid] = [] if uid not in result.keys(): self.logger.error( 'Failed to get flags for mail with uid={}: {}'.format( uid, result)) return self.Retval(False, None) for flag in result[uid]: flags[uid].append(flag.decode('utf-8')) return self.Retval(True, flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def set_mailflags(self, uids, mailbox, flags=[]): """ Set and retrieve flags from mails """ if self.test: self.logger.info( 'Would have set mail flags on message uids "{}"'.format( str(uids))) return self.Retval(True, None) else: self.logger.debug('Setting flags={} on mails uid={}', flags, uids) try: result = self.conn.set_flags(uids, flags) _flags = {} for uid in uids: _flags[uid] = [] if uid not in result.keys(): self.logger.error( 'Failed to set and get flags for mail with uid={}: {}' .format(uid, result)) return self.Retval(False, None) for flag in result[uid]: _flags[uid].append(flag.decode('utf-8')) return self.Retval(True, _flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def add_mailflags(self, uids, mailbox, flags=[]): """ Add and retrieve flags from mails """ if self.test: self.logger.info( 'Would have added mail flags on message uids "{}"'.format( str(uids))) return self.Retval(True, None) else: self.logger.debug('Adding flags={} on mails uid={}', flags, uids) try: result = self.conn.add_flags(uids, flags) _flags = {} for uid in uids: _flags[uid] = [] if uid not in result.keys(): self.logger.error( 'Failed to add and get flags for mail with uid={}: {}' .format(uid, result)) return self.Retval(False, None) for flag in result[uid]: _flags[uid].append(flag.decode('utf-8')) return self.Retval(True, _flags) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def move_mail(self, message_ids, source, destination, delete_old=True, expunge=True, add_flags=None, set_flags=None): """ Move a mail from a mailbox to another """ return self.copy_mails(message_ids=message_ids, source=source, destination=destination, delete_old=delete_old, expunge=expunge, add_flags=add_flags, set_flags=set_flags) @do_select_mailbox def copy_mails(self, source, destination, message_ids=None, delete_old=False, expunge=False, add_flags=None, set_flags=None): """ Copies one or more mails from a mailbox into another """ if self.test: if delete_old: self.logger.info( 'Would have moved mail Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode' .format(message_ids, source, destination)) else: self.logger.info( 'Would have copied mails with Message-Ids="{}" from "{}" to "{}", skipping because of beeing in testmode' .format(message_ids, source, destination)) return self.Retval(True, None) else: try: if delete_old: self.logger.debug( 'Moving mail Message-Ids="{}" from "{}" to "{}"'. format(message_ids, source, destination)) else: self.logger.debug( 'Copying mail Message-Ids="{}" from "{}" to "{}"'. format(message_ids, source, destination)) # if message_ids is None: # message_ids = [] # result = self.fetch_mails(uids=uids, mailbox=source) # if not result.code: # self.logger.error('Failed to determine Message-Id by uids for mail with uids "{}"', uids) # return result # message_ids.append(result.data.keys()) if not self.mailbox_exists(destination).data: self.logger.info( 'Destination mailbox {} doesn\'t exist, creating it for you' .format(destination)) result = self.create_mailbox(mailbox=destination) if not result.code: self.logger.error( 'Failed to create the mailbox {}: {}'.format( source, result.data)) # pragma: no cover return result # pragma: no cover uids = [] for message_id in message_ids: result = self.search_mails( mailbox=source, criteria='HEADER Message-Id "{}"'.format(message_id)) if not result.code or len(result.data) == 0: self.logger.error( 'Failed to determine uid by Message-Id for mail with Message-Id "{}"' .format(message_id)) return self.Retval(False, result.data) uids.append(result.data[0]) result = self.select_mailbox(source) if not result.code: return result # pragma: no cover self.conn.copy(uids, destination) if delete_old: result = self.delete_mails(uids=uids, mailbox=source) if not result.code: self.logger.error( 'Failed to remove old mail with Message-Id="{}"/uids="{}": {}' .format(message_ids, uids, result.data)) # pragma: no cover return result # pragma: no cover if expunge: # TODO don't expunge by default result = self.expunge(mailbox=source) if not result.code: self.logger.error( 'Failed to expunge on mailbox {}: {}'.format( source, result.data)) # pragma: no cover return result # pragma: no cover dest_uids = [] for message_id in message_ids: result = self.search_mails( mailbox=destination, criteria='HEADER Message-Id "{}"'.format(message_id)) if not result.code: self.logger.error( 'Failed to determine uid by Message-Id for mail with Message-Id "{}"' .format(message_id)) # pragma: no cover return result # pragma: no cover dest_uids.append(result.data[0]) if isinstance(set_flags, list): self.set_mailflags(uids=dest_uids, mailbox=destination, flags=set_flags) if add_flags: self.add_mailflags(uids=dest_uids, mailbox=destination, flags=add_flags) return self.Retval(True, dest_uids) except IMAPClient.Error as e: return self.process_error(e) @do_select_mailbox def expunge(self, mailbox): """ Expunge mails form a mailbox """ self.logger.debug('Expunge mails from mailbox {}'.format(mailbox)) try: return self.Retval(True, b'Expunge completed.' in self.conn.expunge()) except IMAPClient.Error as e: # pragma: no cover return self.process_error(e) # pragma: no cover def create_mailbox(self, mailbox): """ Create a mailbox """ self.logger.debug('Creating mailbox {}'.format(mailbox)) try: return self.Retval( True, self.conn.create_folder(mailbox) == b'Create completed.') except IMAPClient.Error as e: return self.process_error(e) def mailbox_exists(self, mailbox): """ Check whether a mailbox exists """ try: return self.Retval(True, self.conn.folder_exists(mailbox)) except IMAPClient.Error as e: # pragma: no cover return self.process_error(e) # pragma: no cover @do_select_mailbox def delete_mails(self, uids, mailbox): """ Delete mails """ self.logger.debug('Deleting mails with uid="{}"'.format(uids)) try: result = self.conn.delete_messages(uids) flags = {} for uid in uids: flags[uid] = [] if uid not in result.keys(): self.logger.error( 'Failed to get flags for mail with uid={} after deleting it: {}' .format(uid, result)) return self.Retval(False, None) for flag in result[uid]: flags[uid].append(flag.decode('utf-8')) return self.Retval(True, flags) except IMAPClient.Error as e: return self.process_error(e)
def check_mbox(self, mailbox): server = IMAPClient(self.host) server.login(self.user, self.password) return server.folder_exists(mailbox)
class MailGetter(object): """ Encapsulates behavior of retrieving email, saving PDFs, and moving emails. """ def __init__(self): """ Class initializer. """ mimetypes.init() logger = Logger() self.logger = logger.get_logger(MAIN + ".Cls") self.server = None self.queue = BotQueue() if not _check_paths(['input_path']): self.logger.fatal("File paths could not be created...cannot continue.") exit() def connect(self) -> bool: """ Connect and login to the remote IMAP server. Returns: (bool): True if successful, otherwise False """ mailserver = os.environ.get('mailserver') imapport = os.environ.get('imapport') mailssl = os.environ.get('mailssl') try: self.server = IMAPClient( mailserver, port=imapport, ssl=mailssl, use_uid=True ) username = os.environ.get('mail_username') password = os.environ.get('mail_password') self.logger.debug(f"Username: {username}, Password: {password}") self.server.login(username, password) except ConnectionRefusedError as e: self.logger.fatal(f"Connection to {mailserver}:{imapport} was refused: {str(e)}") return False except Exception as e: self.logger.fatal(f"Connection to {mailserver}:{imapport} was refused: {str(e)}") return False return True def reconnect(self) -> bool: """ Reconnect to the email server. Returns: (bool): True if successful, otherwise False """ self.logger.debug("Reconnecting to server.") try: self.server.idle_done() except Exception as e: self.logger.error("Error exiting IDLE mode: %s", e) self.logger.info("Will try to reconnect anyway.") try: self.server.logout() except Exception as e: self.logger.error("Error disconnecting from IMAP server: %s", e) self.logger.info("Will try to reconnect anyway.") return self.connect() def check_folders(self) -> bool: """ Check to see if we have the required INBOX and PROCESSED folders. If we do not have an inbox, we can't go on. If we do not have a processed folder, try to create it. Returns: (bool): True if successful, otherwise false. """ # First, make sure the INBOX folder exists. If it does not exist, # we have a serious problem and need to quit. inbox = os.environ.get('inbox') if not self.server.folder_exists(inbox): self.logger.fatal(f"Error locating INBOX named '{inbox}'.") return False # Next, see if the PROCESSED folder exists. If it does not, try to # create it. If we try to create it and the creation fails, again, # we have a serious problem and cannot continue. processed_folder = os.environ.get('processed_folder') if not self.server.folder_exists(processed_folder): self.logger.error(f"Error locating PROCESSED folder named '{processed_folder}'.") self.logger.error(f"Will attempt to create folder named '{processed_folder}'.") try: message = self.server.create_folder(processed_folder) self.logger.info( "Successfully created '%s': %s", processed_folder, message ) except Exception: self.logger.fatal( "Failed to create '%s': %s", processed_folder, message ) return False self.logger.info("Folder check was successful.") return True def save_linked_files( self, links: list, msgid: str, from_email: str, subject: str, reply_to: str ): """ Download and save files referenced by a link instead of directly attached to the email message. Raises exceptions rather than catches them so that the caller's error handler can deal with and record the error. Setting *allow_redirects* to ```True``` let's us retrieve files from cloud services such as DropBox and from URL smashers like www.tinyurl.com. Args: links (list): List of links to download from. msgid (str): ID of the email message we are processing. Used for filename disambiguation. from_email (str): Apparent sender of the email. subject (str): Subject line of the email. reply_to (str): Reply-To Address of the email """ input_path = os.environ.get('input_path') for link in links: if link[-4:].upper() == ".PDF": my_link = cloudize_link(link) self.logger.debug("Found link: %s", my_link) content = requests.get(my_link, allow_redirects=True).content filename = "{}/{}-{}".format( input_path, msgid, urllib.parse.unquote(link[link.rfind("/")+1:]) ) with open(filename, "wb") as fp: fp.write(content) self.queue.publish(arrival_notification( from_email, reply_to, subject, filename, 'application/pdf') ) def process_message(self, msgid, message) -> bool: """ Process one message. Args: msgid (str): Unique ID for this message. message (email): Email message to process. Returns: (bool): True if successful, otherwise False """ self.logger.debug( "ID #%s: From: %s; Subject: %s", msgid, message.get("From"), message.get("Subject") ) from_email = sanitize_from_name(message.get("From")) reply_to = sanitize_from_name(message.get("Return-Path") or from_email) input_path = os.environ.get('input_path') try: subject = message.get("Subject") except OSError as e: self.logger.error("Cannot extract message subject: %s", str(e)) subject = "N/A" for key in message.keys(): self.logger.debug("%s = %s", key, message.get(key, None)) # Go through each part of the message. If we find a pdf file, save it. counter = 1 # number of attachments we've processed for this message. for part in message.walk(): try: # multipart/* are just containers...skip them if part.get_content_maintype() == "multipart": continue # Extract & sanitize filename. Create a filename if not given. filename = part.get_filename() self.logger.debug("*** %s ***", filename) extension = mimetypes.guess_extension(part.get_content_type())\ or ".bin" if not filename: filename = "{}-{}-part-{}{}".format( msgid, from_email, counter, extension ) else: filename = "{}-{}"\ .format(msgid, filename)\ .replace("\r", "")\ .replace("\n", "") counter += 1 # Save the attached file # For now, only save files of type ".PDF" # TODO: Parse HTML parts to see if we have links to PDF files # stored elsewhere. lower_extension = extension.lower() # Save attached file . . . if lower_extension in ['.pdf', '.docx', '.doc', '.rtf']: try: mimetype = mimetypes.types_map[lower_extension] except KeyError as error: self.logger.error( "Cannot map '%s' to a mime type: %s", lower_extension, str(error) ) mimetype = None if mimetype is not None: filename = "{}/{}".format( input_path, filename ) with open(filename, "wb") as fp: fp.write(part.get_payload(decode=True)) self.queue.publish(arrival_notification( from_email, reply_to, subject, filename, mimetype) ) # Save file referenced by a link . . . elif lower_extension[:4] == ".htm": links = extract_html_links(part.get_payload()) self.save_linked_files( links, msgid, from_email, subject, reply_to ) elif lower_extension == ".bat": links = extract_text_links(part.get_payload()) self.save_linked_files( links, msgid, from_email, subject, reply_to ) else: self.logger.info("Skipping: %s", filename) except Exception as e: self.logger.error( "Error with attachment #%s from message #%s from %s: %s", counter, msgid, message.get("From"), e ) self.logger.exception(e) return False return True def wait_for_messages(self): # FIRST: Process anything that's already in our INBOX self.process_inbox() # NEXT: Go into IDLE mode waiting for either a timeout or new mail. self.server.idle() stay_alive = True # Number of times we've returned from idle without receiving any # messages. idle_counter = 0 # Number of seconds to wait for more messages before timing out. idle_timeout = 60 # If we don't receive something within this many seconds, we'll # reconnect to the server. reconnect_seconds = 300 while stay_alive: try: # Wait for up to *idle_timeout* seconds for new messages. responses = self.server.idle_check(timeout=idle_timeout) self.logger.debug("Response to idle_check(): %s", responses) if responses: # We DID get new messages. Process them. idle_counter = 0 responses = self.server.idle_done() self.logger.debug("Response to idle_done(): %s", responses) self.process_inbox() self.server.idle() else: # We did not get any new messages. idle_counter += 1 # If we've run out of patience, reconnect and resume idle # mode. if idle_counter * idle_timeout > reconnect_seconds: if self.reconnect(): idle_counter = 0 self.process_inbox() self.server.idle() # Reconnect failure (!) else: stay_alive = False except imaplib.IMAP4.abort as e: self.logger.error("IMAP connection closed by host: %s", e) self.logger.error("Will try to reconnect") if self.reconnect(): idle_counter = 0 self.process_inbox() self.server.idle() else: self.logger.error("Unable to reconnect. Shutting down.") stay_alive = False except Exception as e: self.logger.error("Error in IDLE loop: %s", e) self.logger.exception(e) self.logger.error("Shutting down due to the above errors.") stay_alive = False def process_inbox(self): """ Process each message in the INBOX. After a message is processed: If processing was successful: Mark as "SEEN" so that we don't process it again. If processing was not successful: Mark as "SEEN" and "FOLLOWUP" so that an operator can fix if and requeue it by clearing the SEEN flag. """ select_info = self.server.select_folder(os.environ.get('inbox')) self.logger.debug( "%d messages in %s.", select_info[b'EXISTS'], os.environ.get('inbox') ) messages = self.server.search(criteria='UNSEEN') for msgid, data in self.server.fetch(messages, ['RFC822']).items(): email_message = email.message_from_bytes(data[b'RFC822']) if self.process_message(msgid, email_message): # Mark message as "Seen". For now, we *won't* move the message # to the PROCESSED folder. self.server.set_flags(msgid, b'\\Seen') else: # If we had an error processing the message, mark it for # follow AND as seen. self.server.set_flags( msgid, [b'\\Flagged for Followup', b'\\Seen'] )
class MailToolbox(SourceFactory): def __init__(self, hote_imap, nom_utilisateur, mot_de_passe, dossier_cible='INBOX', verify_peer=True, use_secure_socket=True, legacy_secure_protocol=False): super().__init__('IMAPFactory via {}'.format(hote_imap)) self._ssl_context = SSLContext( protocol=PROTOCOL_TLS) if use_secure_socket else None self._use_secure_socket = use_secure_socket if verify_peer is False and use_secure_socket is True: # don't check if certificate hostname doesn't match target hostname self._ssl_context.check_hostname = False # don't check if the certificate is trusted by a certificate authority self._ssl_context.verify_mode = CERT_NONE if legacy_secure_protocol and use_secure_socket: self._ssl_context.options = OP_ALL self._hote_imap = Session.UNIVERSELLE.retranscrire(hote_imap) self._client = IMAPClient(host=self._hote_imap, port=993 if self._use_secure_socket else 143, ssl=self._use_secure_socket, ssl_context=self._ssl_context) self._nom_utilisateur = Session.UNIVERSELLE.retranscrire( nom_utilisateur) self._verify_peer = verify_peer self._dossier_cible = dossier_cible self._mot_de_passe = mot_de_passe self._client.login( self._nom_utilisateur, Session.UNIVERSELLE.retranscrire(self._mot_de_passe)) self._echec = False MailToolbox.INSTANCES.append(self) @property def est_hors_service(self): return self._echec def reset(self): try: self._client.logout() except IMAPClientError as e: pass except OSError as e: pass try: self._client = IMAPClient( host=self._hote_imap, port=993 if self._use_secure_socket else 143, ssl=self._use_secure_socket, ssl_context=self._ssl_context) self._client.login( self._nom_utilisateur, Session.UNIVERSELLE.retranscrire(self._mot_de_passe)) except IMAPClientError as e: logger.error( "Une erreur IMAP critique est survenue lors de la reconnexion. {msg_err}", msg_err=str(e)) self._echec = True return self._echec = False @property def dossier_cible(self): return self._dossier_cible @dossier_cible.setter def dossier_cible(self, nouveau_dossier_cible): if isinstance(nouveau_dossier_cible, str): self._dossier_cible = nouveau_dossier_cible @property def hote_imap(self): return self._hote_imap @property def nom_utilisateur(self): return self._nom_utilisateur @staticmethod def fetch_instance(hote_imap, nom_utilisateur): hote_imap, nom_utilisateur = Session.UNIVERSELLE.retranscrire( hote_imap), Session.UNIVERSELLE.retranscrire(nom_utilisateur) for inst in MailToolbox.INSTANCES: if isinstance( inst, MailToolbox ) is True and inst.nom_utilisateur == nom_utilisateur and inst.hote_imap == hote_imap: return inst return None def extraire(self, no_progress_bar=True): if self.est_hors_service is True: self.reset() try: self._client.select_folder(self._dossier_cible) except IMAPClientError as e: raise ExtractionSourceException('IMAPClientError: ' + str(e)) except IMAPClientAbortError as e: raise ExtractionSourceException('IMAPClientAbortError: ' + str(e)) except IMAPClientReadOnlyError as e: raise ExtractionSourceException('IMAPClientReadOnlyError: ' + str(e)) finally: self._echec = True # fetch selectors are passed as a simple list of strings. responses = self._client.fetch(self._client.search(['NOT', 'DELETED']), ['UID', 'ENVELOPE', 'BODY', 'RFC822']) extractions = list() # type: list[Mail] for id_response in tqdm(responses.keys( ), unit=' message') if no_progress_bar is False else responses.keys(): email_message = email.message_from_bytes( responses[id_response][b'RFC822']) mail = Mail.from_message(email_message) mail.folder = self._dossier_cible mail.flags = responses[id_response][ b'FLAGS'] if b'FLAGS' in responses[id_response].keys( ) else tuple() mail.bal_internal_id = id_response extractions.append(mail) mail.factory = self return sorted(extractions, key=lambda x: x.date_received or datetime.now()) def copier(self, mail, dossier_dest): """ :param Mail mail: :param str dossier_dest: :return: """ if self.est_hors_service is True: self.reset() try: if self._client.folder_exists(dossier_dest) is False: raise FileNotFoundError( 'Le dossier "{}" n\'existe pas sur le serveur IMAP distant !' ) self._client.select_folder(mail.folder) self._client.copy([mail.bal_internal_id], dossier_dest) except IMAPClientError as e: raise ManipulationSourceException('IMAPClientError: ' + str(e)) except IMAPClientAbortError as e: raise ManipulationSourceException('IMAPClientAbortError: ' + str(e)) except IMAPClientReadOnlyError as e: raise ManipulationSourceException('IMAPClientReadOnlyError: ' + str(e)) finally: self._echec = True def deplacer(self, mail, dossier_dest): if self.est_hors_service is True: self.reset() try: if self._client.folder_exists(dossier_dest) is False: raise FileNotFoundError( 'Le dossier "{}" n\'existe pas sur le serveur IMAP distant !' ) self._client.select_folder(mail.folder) try: self._client.move([mail.bal_internal_id], dossier_dest) except CapabilityError as e: self.copier(mail, dossier_dest) self.supprimer(mail) except IMAPClientError as e: raise ManipulationSourceException('IMAPClientError: ' + str(e)) except IMAPClientAbortError as e: raise ManipulationSourceException('IMAPClientAbortError: ' + str(e)) except IMAPClientReadOnlyError as e: raise ManipulationSourceException('IMAPClientReadOnlyError: ' + str(e)) finally: self._echec = True def supprimer(self, mail): if self.est_hors_service is True: self.reset() try: self._client.select_folder(mail.folder) self._client.delete_messages([mail.bal_internal_id], silent=True) self._client.expunge([mail.bal_internal_id]) except IMAPClientError as e: raise ManipulationSourceException('IMAPClientError: ' + str(e)) except IMAPClientAbortError as e: raise ManipulationSourceException('IMAPClientAbortError: ' + str(e)) except IMAPClientReadOnlyError as e: raise ManipulationSourceException('IMAPClientReadOnlyError: ' + str(e)) finally: self._echec = True
class ImapConfiguration(DataRetriever): def __init__(self, instrument_id, logger, configuration=None): super().__init__(instrument_id, logger) if configuration is None: self.configuration["Server"] = None self.configuration["Port"] = None self.configuration["User"] = None self.configuration["Password"] = None self.configuration["Source Folder"] = None self.configuration["Downloaded Folder"] = None else: self.configuration = configuration # For storing the current IMAP connection self.imapconn = None # For storing the current message ID during processing self.current_id = None @staticmethod def get_type(): return "IMAP Email" # Check that the configuration works def test_configuration(self): config_ok = True logged_in = False # Connect try: self.imapconn = IMAPClient(host=self.configuration["Server"], port=self.configuration["Port"]) except: self.log( logging.CRITICAL, "Cannot connect to IMAP server: " + traceback.format_exc()) config_ok = False # Authenticate try: if self.imapconn is not None: self.imapconn.login(self.configuration["User"], self.configuration["Password"]) logged_in = True # Check folders if not self.imapconn.folder_exists( self.configuration["Source Folder"]): self.log(logging.CRITICAL, "Source Folder does not exist") config_ok = False if not self.imapconn.folder_exists( self.configuration["Downloaded Folder"]): self.log(logging.CRITICAL, "Downloaded Folder does not exist") config_ok = False except: self.log(logging.CRITICAL, "Cannot log in to IMAP server: " + traceback.format_exc()) config_ok = False # Shut everything down self.shutdown() return config_ok # Initialise a connection to the IMAP server def startup(self): result = True try: # Log in to the mail server self.imapconn = IMAPClient(host=self.configuration["Server"], port=self.configuration["Port"]) self.imapconn.login(self.configuration["User"], self.configuration["Password"]) # Get the list of messages we can process self.imapconn.select_folder(self.configuration["Source Folder"]) self.message_ids = self.imapconn.search(["NOT", "DELETED"]) self.current_index = -1 except: self.log(logging.CRITICAL, "Cannot log in to IMAP server: " + traceback.format_exc()) result = False return result # Shutdown the server connection def shutdown(self): if self.imapconn is not None: try: if logged_in: self.imapconn.logout() self.imapconn.shutdown() except: # Don't care pass self.imapconn = None # Get the next message and extract its attachment def _retrieve_next_file(self): try: file_found = False self.current_index = self.current_index + 1 while not file_found and self.current_index < len( self.message_ids): self.log(logging.DEBUG, "Processing email ID " + \ str(self.message_ids[self.current_index])) message_content = self.imapconn.fetch( \ self.message_ids[self.current_index], "RFC822") \ [self.message_ids[self.current_index]][b"RFC822"] message = email.message_from_bytes(message_content) for part in message.walk(): content_disposition = part.get("Content-Disposition") if content_disposition is not None and \ content_disposition.startswith("attachment"): filename = self._extract_filename(part.get_filename()) self.log(logging.DEBUG, "Extracting attachment " + filename) contents = part.get_payload(decode=True) self._add_file(filename, contents) file_found = True if not file_found: self.imapconn.move(self.message_ids[self.current_index], self.configuration["Downloaded Folder"]) self.current_index = self.current_index + 1 except: self.log(logging.ERROR, "Failed to retrieve next file: " + traceback.format_exc()) # Extract a filename, handling encoded filenames as required def _extract_filename(self, filename): result = filename if decode_header(filename)[0][1] is not None: result = decode_header(filename)[0][0].decode( decode_header(filename)[0][1]) return result # The file(s) have been processed successfully; # clean them up accordingly def _cleanup_success(self): try: self.log( logging.DEBUG, "Moving email " + str(self.message_ids[self.current_index]) + " to downloaded folder") self.imapconn.move(self.message_ids[self.current_index], self.configuration["Downloaded Folder"]) except: self.log( logging.ERROR, "Failed to move email after processing: " + traceback.format_exc()) # The file(s) were not processed successfully; # clean them up accordingly def _cleanup_fail(self): # We don't do anything - we'll try to process # the mail again next time round pass # The file(s) were not processed def _cleanup_not_processed(self): # No action required pass
class MailFetcher(object): def __init__(self, host, user, password, folder_inbox, folder_archive, folder_error): self.server = IMAPClient(host) self.server.login(user, password) self.folder_inbox = folder_inbox self.folder_archive = folder_archive self.folder_error = folder_error if self.server.folder_exists(self.folder_archive): logger.info("Archive folder '{folder}' exists.".format( folder=self.folder_archive)) else: self.server.create_folder(self.folder_archive) logger.info("Archive folder '{folder}' created.".format( folder=self.folder_archive)) if self.server.folder_exists(self.folder_error): logger.info("Error folder '{folder}' exists.".format( folder=self.folder_error)) else: self.server.create_folder(self.folder_error) logger.info("Error folder '{folder}' created.".format( folder=self.folder_error)) def findIcs(self, rawmail): email_message = email.message_from_bytes(rawmail) for part in email_message.walk(): logger.debug(part.get_content_type()) if ("text/calendar" in part.get_content_type()): logger.info( "Found calendar event in mail from {sender} with subject '{subject}'." .format(sender=email_message.get('From'), subject=email_message.get('Subject'))) return ical_tools.parse(part.get_payload(decode=True)) def getIcs(self): select_info = self.server.select_folder(self.folder_inbox) logger.debug("{num_mails} messages in folder {folder}.".format( num_mails=select_info[b'EXISTS'], folder=self.folder_inbox)) ics_list = [] messages = self.server.search('ALL') for message_uid, message_data in self.server.fetch(messages, 'RFC822').items(): logger.debug("Found message with id {id}".format(id=message_uid)) ics = self.findIcs(message_data[b'RFC822']) if (ics): ics_list.append((message_uid, ics)) return ics_list def archiveMessage(self, uid, error=False): folder = self.folder_archive if error: folder = self.folder_error self.server.move([uid], folder) logger.info("Message with UID {uid} moved to folder {folder}".format( uid=uid, folder=folder))