def headers_transform_encodedword_test(): # Create a header with non-ascii characters that will be stored in encoded-word format. headers = [('Subject', encoding.to_mime('Subject', u'Hello ✓'))] h = MimeHeaders(headers) # transform should decode it for us when we pass decode=True h.transform(lambda key,val: (key, val.replace(u'✓', u'☃')), decode=True) eq_(u'Hello ☃', h.get('Subject'))
def headers_length_test(): h = MimeHeaders() eq_(0, len(h)) headers = [('mime-version', '1'), ('rEceived', '2'), ('mime-version', '3'), ('ReceiveD', '4')] h = MimeHeaders(headers) eq_(4, len(h))
def headers_transform_encodedword_test(): # Create a header with non-ascii characters that will be stored in encoded-word format. headers = [('Subject', encoding.to_mime('Subject', u'Hello ✓'))] h = MimeHeaders(headers) # transform should decode it for us when we pass decode=True h.transform(lambda key, val: (key, val.replace(u'✓', u'☃')), decode=True) eq_(u'Hello ☃', h.get('Subject'))
def headers_case_insensitivity_test(): h = MimeHeaders() h['Content-Type'] = 1 eq_(1, h['Content-Type']) eq_(1, h['conTenT-TyPE']) ok_('cOnTenT-TyPE' in h) ok_('Content-Type' in h) eq_(1, h.get('Content-Type')) eq_(None, h.get('Content-Type2')) eq_([('Content-Type', 1)], h.items())
def headers_case_insensitivity_test(): h = MimeHeaders() h["Content-Type"] = 1 eq_(1, h["Content-Type"]) eq_(1, h["conTenT-TyPE"]) ok_("cOnTenT-TyPE" in h) ok_("Content-Type" in h) eq_(1, h.get("Content-Type")) eq_(None, h.get("Content-Type2")) eq_([("Content-Type", 1)], h.items())
def headers_roundtrip_test(): headers = MimeHeaders.from_stream(StringIO(BILINGUAL)) out = StringIO() headers.to_stream(out) headers2 = MimeHeaders.from_stream(StringIO(out.getvalue())) eq_(21, len(headers2)) eq_(u"Simple text. How are you? Как ты поживаешь?", headers["Subject"]) received_headers = headers.getall("Received") eq_(5, len(received_headers)) ok_("c2cs24435ybk" in received_headers[0]) eq_(headers["Content-Transfer-Encoding"], headers2["Content-Transfer-Encoding"]) eq_(headers["DKIM-Signature"], headers2["DKIM-Signature"])
def headers_roundtrip_test(): headers = MimeHeaders.from_stream(StringIO(BILINGUAL)) out = StringIO() headers.to_stream(out) headers2 = MimeHeaders.from_stream(StringIO(out.getvalue())) eq_(21, len(headers2)) eq_(u"Simple text. How are you? Как ты поживаешь?", headers['Subject']) received_headers = headers.getall('Received') eq_(5, len(received_headers)) ok_('c2cs24435ybk' in received_headers[0]) eq_(headers['Content-Transfer-Encoding'], headers2['Content-Transfer-Encoding']) eq_(headers['DKIM-Signature'], headers2['DKIM-Signature'])
def headers_order_preserved_test(): headers = [("mime-version", "1"), ("rEceived", "2"), ("mime-version", "3"), ("ReceiveD", "4")] h = MimeHeaders(headers) # various types of iterations should_be = [("Mime-Version", "1"), ("Received", "2"), ("Mime-Version", "3"), ("Received", "4")] eq_(should_be, h.items()) ok_(isinstance(h.items(), list)) eq_(should_be, [p for p in h.iteritems()]) # iterate over keys keys = ["Mime-Version", "Received", "Mime-Version", "Received"] eq_(keys, [p for p in h]) eq_(keys, h.keys())
def headers_order_preserved_test(): headers = [('mime-version', '1'), ('rEceived', '2'), ('mime-version', '3'), ('ReceiveD', '4')] h = MimeHeaders(headers) # various types of iterations should_be = [('Mime-Version', '1'), ('Received', '2'), ('Mime-Version', '3'), ('Received', '4')] eq_(should_be, h.items()) ok_(isinstance(h.items(), list)) eq_(should_be, [p for p in h.iteritems()]) # iterate over keys keys = ['Mime-Version', 'Received', 'Mime-Version', 'Received'] eq_(keys, [p for p in h]) eq_(keys, h.keys())
def headers_roundtrip_test(): headers = MimeHeaders.from_stream(six.StringIO(BILINGUAL.decode('utf-8'))) out = six.StringIO() headers.to_stream(out) headers2 = MimeHeaders.from_stream(six.StringIO(out.getvalue())) eq_(21, len(headers2)) eq_(u"Simple text. How are you? Как ты поживаешь?", headers['Subject']) received_headers = headers.getall('Received') eq_(5, len(received_headers)) ok_('c2cs24435ybk' in received_headers[0]) eq_(headers['Content-Transfer-Encoding'], headers2['Content-Transfer-Encoding']) eq_(headers['DKIM-Signature'], headers2['DKIM-Signature'])
def bilingual_message_test(): headers = MimeHeaders.from_stream(StringIO(BILINGUAL)) eq_(21, len(headers)) eq_(u"Simple text. How are you? Как ты поживаешь?", headers['Subject']) received_headers = headers.getall('Received') eq_(5, len(received_headers)) ok_('c2cs24435ybk' in received_headers[0])
def headers_order_preserved_test(): headers = [('mime-version', '1'), ('rEceived', '2'), ('mime-version', '3'), ('ReceiveD', '4')] h = MimeHeaders(headers) # various types of iterations should_be = [('Mime-Version', '1'), ('Received', '2'), ('Mime-Version', '3'), ('Received', '4')] eq_(should_be, h.items()) ok_(isinstance(h.items(), list)) eq_(should_be, [p for p in h.iteritems()]) # iterate over keys keys = ['Mime-Version', 'Received', 'Mime-Version', 'Received'] eq_(keys, [p for p in h]) eq_(keys, list(h.keys()))
def collect(message): collected = deque() for p in message.walk(with_self=True): for h in HEADERS: if h in p.headers: collected.append((h, p.headers[h])) if p.content_type.is_delivery_status(): collected += collect_from_status(p.body) return MimeHeaders(collected)
def headers_transform_test(): headers = [('mime-version', '1'), ('rEceived', '2'), ('mime-version', '3'), ('ReceiveD', '4')] h = MimeHeaders(headers) # transform tracks whether anything actually changed h.transform(lambda key, val: (key, val)) assert_false(h.have_changed()) # ok, now something have changed, make sure we've preserved order and did not collapse anything h.transform(lambda key, val: ("X-{0}".format(key), "t({0})".format(val))) ok_(h.have_changed()) eq_([('X-Mime-Version', 't(1)'), ('X-Received', 't(2)'), ('X-Mime-Version', 't(3)'), ('X-Received', 't(4)')], h.items())
def headers_multiple_values_test(): headers = [("mime-version", "1"), ("rEceived", "2"), ("mime-version", "3"), ("ReceiveD", "4")] h = MimeHeaders(headers) eq_(["1", "3"], h.getall("Mime-Version")) # set re-sets all values for the message h["Mime-Version"] = "5" eq_(["5"], h.getall("Mime-Version")) # use add to add more values h.add("Received", "6") eq_(["2", "4", "6"], h.getall("Received")) # use prepend to insert header in the begining of the list h.prepend("Received", "0") eq_(["0", "2", "4", "6"], h.getall("Received")) # delete removes it all! del h["RECEIVED"] eq_([], h.getall("Received"))
def headers_multiple_values_test(): headers = [('mime-version', '1'), ('rEceived', '2'), ('mime-version', '3'), ('ReceiveD', '4')] h = MimeHeaders(headers) eq_(['1', '3'], h.getall('Mime-Version')) # set re-sets all values for the message h['Mime-Version'] = '5' eq_(['5'], h.getall('Mime-Version')) # use add to add more values h.add('Received', '6') eq_(['2', '4', '6'], h.getall('Received')) # use prepend to insert header in the begining of the list h.prepend('Received', '0') eq_(['0', '2', '4', '6'], h.getall('Received')) # delete removes it all! del h['RECEIVED'] eq_([], h.getall('Received'))
def headers_transform_test(): headers = [("mime-version", "1"), ("rEceived", "2"), ("mime-version", "3"), ("ReceiveD", "4")] h = MimeHeaders(headers) # transform tracks whether anything actually changed h.transform(lambda key, val: (key, val)) assert_false(h.have_changed()) # ok, now something have changed, make sure we've preserved order and did not collapse anything h.transform(lambda key, val: ("X-{}".format(key), "t({})".format(val))) ok_(h.have_changed()) eq_( [("X-Mime-Version", "t(1)"), ("X-Received", "t(2)"), ("X-Mime-Version", "t(3)"), ("X-Received", "t(4)")], h.items(), )
def headers_transform_test(): headers = [('mime-version', '1'), ('rEceived', '2'), ('mime-version', '3'), ('ReceiveD', '4')] h = MimeHeaders(headers) # transform tracks whether anything actually changed h.transform(lambda key,val: (key, val)) assert_false(h.have_changed()) # ok, now something have changed, make sure we've preserved order and did not collapse anything h.transform(lambda key,val: ("X-{0}".format(key), "t({0})".format(val))) ok_(h.have_changed()) eq_([('X-Mime-Version', 't(1)'), ('X-Received', 't(2)'), ('X-Mime-Version', 't(3)'), ('X-Received', 't(4)')], h.items())
def headers_alternation_test(): headers = [('mime-version', '1'), ('rEceived', '2'), ('mime-version', '3'), ('ReceiveD', '4')] h = MimeHeaders(headers) assert_false(h.have_changed()) h.prepend('Received', 'Yo') ok_(h.have_changed()) h = MimeHeaders(headers) del h['Mime-Version'] ok_(h.have_changed()) h = MimeHeaders(headers) h['Mime-Version'] = 'a' ok_(h.have_changed()) h = MimeHeaders(headers) h.add('Mime-Version', 'a') ok_(h.have_changed()) h = MimeHeaders(headers) h.getall('Mime-Version') h.get('o') assert_false(h.have_changed())
def headers_parsing_ridiculously_long_line_test(): val = "abcdefg" * 100000 header = "Hello: {0}\r\n".format(val) MimeHeaders.from_stream(StringIO(header))
def headers_alternation_test(): headers = [("mime-version", "1"), ("rEceived", "2"), ("mime-version", "3"), ("ReceiveD", "4")] h = MimeHeaders(headers) assert_false(h.have_changed()) h.prepend("Received", "Yo") ok_(h.have_changed()) h = MimeHeaders(headers) del h["Mime-Version"] ok_(h.have_changed()) h = MimeHeaders(headers) h["Mime-Version"] = "a" ok_(h.have_changed()) h = MimeHeaders(headers) h.add("Mime-Version", "a") ok_(h.have_changed()) h = MimeHeaders(headers) h.getall("Mime-Version") h.get("o") assert_false(h.have_changed())
def add(self, key, value): MimeHeaders.add(self, key, value) self._m[key] = headers.to_mime(normalize(key), remove_newlines(value))
def prepend(self, key, value): MimeHeaders.prepend(self, key, value) self._m._headers.insert(0, (normalize(key), remove_newlines(value)))
def headers_parsing_binary_stuff_survives_test(): value = zlib.compress(b"abcdefg") header = "Hello: {0}\r\n".format(value) ok_(MimeHeaders.from_stream(StringIO(header)))
def transform(self, func): MimeHeaders.transform(self, func) self._m._headers = self.items()
def broken_sequences_test(): headers = StringIO(" hello this is a bad header\nGood: this one is ok") headers = MimeHeaders.from_stream(headers) eq_(1, len(headers)) eq_("this one is ok", headers["Good"])
def slurp_imap_namespace_gmail(imap, db, namespace=None, account=None): # folder attrs -> RFC 6154 Special-Use mailbox flags singleton_flags = { 'all_folder': u'\\All', 'archive_folder': u'\\Archive', 'drafts_folder': u'\\Drafts', 'starred_folder': u'\\Flagged', 'spam_folder': u'\\Junk', 'sent_folder': u'\\Sent', 'trash_folder': u'\\Trash', } # List folders -- Returns sequence of (flags, delimiter, name) folders_fdn = imap.list_folders() with db: # Folder names & delimiters db.executemany( """ INSERT INTO folders ( folder_name, clean_folder_name, imap_delimiter ) VALUES (?, ?, ?) """, ((name, cleanup_folder_name(name), delimiter) for flags, delimiter, name in folders_fdn)) # Folder flags db.executemany( """ INSERT INTO folder_flags (folder_name, flag) VALUES (?, ?) """, ((name, flag) for flags, delimiter, name in folders_fdn for flag in flags)) # Set imap_noselect = 1 on folders that have the \Noselect flag; # Set imap_noselect = 0 on folders that don't. db.execute(""" UPDATE folders SET imap_noselect = ( SELECT folder_flags.flag IS NOT NULL FROM folders AS a LEFT JOIN folder_flags ON ( a.folder_name = folder_flags.folder_name AND folder_flags.flag = '\Noselect' ) WHERE folders.folder_name = a.folder_name ) """) # Insert 'inbox_folder' -> 'INBOX' if there is an INBOX folder, which # there should always be, I think. db.execute( """ INSERT INTO special_folders (attr_name, folder_name) SELECT ?, folder_name FROM folders WHERE folder_name = ? """, ['inbox_folder', 'INBOX']) # Insert other special folder names db.executemany( """ INSERT INTO special_folders (attr_name, folder_name) SELECT ?, folder_name FROM folder_flags WHERE flag = ? """, singleton_flags.items()) # Fetch all messages from each folder with db: folder_names = [ row[0] for row in db.execute( "SELECT folder_name FROM folders WHERE NOT imap_noselect") ] for folder_name in folder_names: # EXAMINE the folder examine_response = imap.select_folder(folder_name, readonly=True) # Update imap_uidvalidity db.execute( """ UPDATE folders SET imap_uidvalidity = ?, imap_uidnext = ? WHERE folder_name = ? """, [ examine_response[u'UIDVALIDITY'], examine_response[u'UIDNEXT'], folder_name ]) # Get uids of the messages in the folder imap_uids = imap.search(u'ALL') # Result should match the stated number of messages in the folder. if len(imap_uids) != examine_response[u'EXISTS']: raise AssertionError("len(imap_uids)={0}, EXISTS={1!r}".format( len(imap_uids), examine_response[u'EXISTS'])) # Create folder_messages entries db.executemany( """ INSERT INTO folder_messages (folder_name, imap_uid) VALUES (?, ?) """, ((folder_name, imap_uid) for imap_uid in imap_uids)) ## Get the folder flags #folder_flags = set(row[0] for row in db.execute( # "SELECT flag FROM folder_flags WHERE folder_name = ?", # [folder_name])) # ## This is Gmail, so only actually fetch messages from the 'All ## Mail' and 'Trash' folders. This *should* give us all of the ## messages. #if not folder_flags & {u'\\All', u'\\Trash', u'\\Sent'}: # continue # Get folder messages batch_size = 1000 fetch_data = [ 'RFC822.SIZE', 'ENVELOPE', 'FLAGS', 'X-GM-MSGID', 'X-GM-THRID', 'X-GM-LABELS', 'INTERNALDATE', 'RFC822.HEADER' ] for i in range(0, len(imap_uids), batch_size): imap_uids_batch = imap_uids[i:i + batch_size] # Fetch message info from the IMAP server fetch_response = imap.fetch(imap_uids_batch, fetch_data) # Fetch message info and insert it into the messages table. # Don't bother deduplicating at this point. for uid, data in fetch_response.items(): headers = MimeHeaders.from_stream( StringIO(data['RFC822.HEADER'])) msg_data = dict( date=data['INTERNALDATE'], subject=data['ENVELOPE'].subject, in_reply_to=data['ENVELOPE'].in_reply_to, size=data['RFC822.SIZE'], message_id_header=data['ENVELOPE'].message_id, x_gm_thrid=unicode(data['X-GM-THRID']), x_gm_msgid=unicode(data['X-GM-MSGID']), sender_addr=json.dumps( parse_email_address_list(headers.get('Sender'))), from_addr=json.dumps( parse_email_address_list(headers.get('From'))), reply_to_addr=json.dumps( parse_email_address_list(headers.get('Reply-To'))), to_addr=json.dumps( parse_email_address_list(headers.get('To'))), cc_addr=json.dumps( parse_email_address_list(headers.get('Cc'))), bcc_addr=json.dumps( parse_email_address_list(headers.get('Bcc'))), ) msg_data['clean_subject'] = \ cleanup_subject(parse_header_value('Subject', msg_data['subject'])) # Check if we've already stored the message cur = db.execute( """ SELECT id, x_gm_msgid FROM messages WHERE x_gm_msgid = :x_gm_msgid """, msg_data) row = next(iter(cur.fetchall()), None) # returns 0 or 1 rows message_id = row['id'] if row is not None else None # If we've never stored the message, store it now. if message_id is None: cur = db.execute( """ INSERT INTO messages ( date, subject, clean_subject, in_reply_to, size, message_id_header, x_gm_msgid, x_gm_thrid, sender_addr, from_addr, reply_to_addr, to_addr, cc_addr, bcc_addr ) VALUES ( :date, :subject, :clean_subject, :in_reply_to, :size, :message_id_header, :x_gm_msgid, :x_gm_thrid, :sender_addr, :from_addr, :reply_to_addr, :to_addr, :cc_addr, :bcc_addr ) """, msg_data) message_id = cur.lastrowid # Store the Gmail labels (these can be different in # different folders; e.g. messages in the 'Sent' folder are # missing the u'\\Sent' label) db.executemany( """ INSERT INTO folder_message_gm_labels (folder_name, message_id, label) VALUES (?, ?, ?) """, ((folder_name, message_id, label) for label in data['X-GM-LABELS'])) # Mark the message as being in the current folder. db.execute( """ UPDATE folder_messages SET message_id = ? WHERE folder_name = ? AND imap_uid = ? """, (message_id, folder_name, uid)) # Construct threads (assuming gmail for now) db.execute(""" INSERT INTO threads (x_gm_thrid) SELECT DISTINCT x_gm_thrid FROM messages """) db.execute(""" INSERT INTO thread_messages (thread_id, message_id) SELECT threads.id, messages.id FROM threads, messages WHERE threads.x_gm_thrid = messages.x_gm_thrid """) # Construct folder_threads db.execute(""" INSERT INTO folder_threads (folder_name, thread_id) SELECT DISTINCT folder_messages.folder_name, thread_messages.thread_id FROM folder_messages LEFT JOIN thread_messages USING (message_id) """)
def __delitem__(self, key): MimeHeaders.__delitem__(self, key) del self._m[key]
def headers_parsing_ridiculously_long_line_test(): val = "abcdefg"*100000 header = "Hello: {0}\r\n".format(val) MimeHeaders.from_stream(StringIO(header))
def headers_multiple_values_test(): headers = [('mime-version', '1'), ('rEceived', '2'), ('mime-version', '3'), ('ReceiveD', '4')] h = MimeHeaders(headers) eq_(['1', '3'], h.getall('Mime-Version')) # set re-sets all values for the message h['Mime-Version'] = '5' eq_(['5'], h.getall('Mime-Version')) # use add to add more values h.add('Received', '1') eq_(['1', '2', '4'], h.getall('Received')) # use prepend to insert header in the begining of the list h.prepend('Received', '0') eq_(['0', '1', '2', '4'], h.getall('Received')) # delete removes it all! del h['RECEIVED'] eq_([], h.getall('Received'))
def __setitem__(self, key, value): MimeHeaders.__setitem__(self, key, value) del self._m[key] self._m[key] = remove_newlines(value)
def test_folding_combinations(): message = """From [email protected] Mon Feb 8 02:53:47 PST 1993\nTo: sasha\r\n continued\n line\nFrom: single line \r\nSubject: hello, how are you\r\n today?""" headers = MimeHeaders.from_stream(StringIO(message)) eq_('sasha continued line', headers['To']) eq_('single line ', headers['From']) eq_("hello, how are you today?", headers['Subject'])
def headers_parsing_empty_test(): h = MimeHeaders.from_stream(StringIO("")) eq_(0, len(h))
def slurp_imap_namespace_gmail(imap, db, namespace=None, account=None): # folder attrs -> RFC 6154 Special-Use mailbox flags singleton_flags = { 'all_folder': u'\\All', 'archive_folder': u'\\Archive', 'drafts_folder': u'\\Drafts', 'starred_folder': u'\\Flagged', 'spam_folder': u'\\Junk', 'sent_folder': u'\\Sent', 'trash_folder': u'\\Trash', } # List folders -- Returns sequence of (flags, delimiter, name) folders_fdn = imap.list_folders() with db: # Folder names & delimiters db.executemany(""" INSERT INTO folders ( folder_name, clean_folder_name, imap_delimiter ) VALUES (?, ?, ?) """, ((name, cleanup_folder_name(name), delimiter) for flags, delimiter, name in folders_fdn)) # Folder flags db.executemany(""" INSERT INTO folder_flags (folder_name, flag) VALUES (?, ?) """, ((name, flag) for flags, delimiter, name in folders_fdn for flag in flags)) # Set imap_noselect = 1 on folders that have the \Noselect flag; # Set imap_noselect = 0 on folders that don't. db.execute(""" UPDATE folders SET imap_noselect = ( SELECT folder_flags.flag IS NOT NULL FROM folders AS a LEFT JOIN folder_flags ON ( a.folder_name = folder_flags.folder_name AND folder_flags.flag = '\Noselect' ) WHERE folders.folder_name = a.folder_name ) """) # Insert 'inbox_folder' -> 'INBOX' if there is an INBOX folder, which # there should always be, I think. db.execute(""" INSERT INTO special_folders (attr_name, folder_name) SELECT ?, folder_name FROM folders WHERE folder_name = ? """, ['inbox_folder', 'INBOX']) # Insert other special folder names db.executemany(""" INSERT INTO special_folders (attr_name, folder_name) SELECT ?, folder_name FROM folder_flags WHERE flag = ? """, singleton_flags.items()) # Fetch all messages from each folder with db: folder_names = [row[0] for row in db.execute( "SELECT folder_name FROM folders WHERE NOT imap_noselect")] for folder_name in folder_names: # EXAMINE the folder examine_response = imap.select_folder(folder_name, readonly=True) # Update imap_uidvalidity db.execute(""" UPDATE folders SET imap_uidvalidity = ?, imap_uidnext = ? WHERE folder_name = ? """, [examine_response[u'UIDVALIDITY'], examine_response[u'UIDNEXT'], folder_name]) # Get uids of the messages in the folder imap_uids = imap.search(u'ALL') # Result should match the stated number of messages in the folder. if len(imap_uids) != examine_response[u'EXISTS']: raise AssertionError("len(imap_uids)={0}, EXISTS={1!r}".format( len(imap_uids), examine_response[u'EXISTS'])) # Create folder_messages entries db.executemany(""" INSERT INTO folder_messages (folder_name, imap_uid) VALUES (?, ?) """, ((folder_name, imap_uid) for imap_uid in imap_uids)) ## Get the folder flags #folder_flags = set(row[0] for row in db.execute( # "SELECT flag FROM folder_flags WHERE folder_name = ?", # [folder_name])) # ## This is Gmail, so only actually fetch messages from the 'All ## Mail' and 'Trash' folders. This *should* give us all of the ## messages. #if not folder_flags & {u'\\All', u'\\Trash', u'\\Sent'}: # continue # Get folder messages batch_size = 1000 fetch_data = ['RFC822.SIZE', 'ENVELOPE', 'FLAGS', 'X-GM-MSGID', 'X-GM-THRID', 'X-GM-LABELS', 'INTERNALDATE', 'RFC822.HEADER'] for i in range(0, len(imap_uids), batch_size): imap_uids_batch = imap_uids[i:i+batch_size] # Fetch message info from the IMAP server fetch_response = imap.fetch(imap_uids_batch, fetch_data) # Fetch message info and insert it into the messages table. # Don't bother deduplicating at this point. for uid, data in fetch_response.items(): headers = MimeHeaders.from_stream(StringIO(data['RFC822.HEADER'])) msg_data = dict( date=data['INTERNALDATE'], subject=data['ENVELOPE'].subject, in_reply_to=data['ENVELOPE'].in_reply_to, size=data['RFC822.SIZE'], message_id_header=data['ENVELOPE'].message_id, x_gm_thrid=unicode(data['X-GM-THRID']), x_gm_msgid=unicode(data['X-GM-MSGID']), sender_addr=json.dumps(parse_email_address_list(headers.get('Sender'))), from_addr=json.dumps(parse_email_address_list(headers.get('From'))), reply_to_addr=json.dumps(parse_email_address_list(headers.get('Reply-To'))), to_addr=json.dumps(parse_email_address_list(headers.get('To'))), cc_addr=json.dumps(parse_email_address_list(headers.get('Cc'))), bcc_addr=json.dumps(parse_email_address_list(headers.get('Bcc'))), ) msg_data['clean_subject'] = \ cleanup_subject(parse_header_value('Subject', msg_data['subject'])) # Check if we've already stored the message cur = db.execute(""" SELECT id, x_gm_msgid FROM messages WHERE x_gm_msgid = :x_gm_msgid """, msg_data) row = next(iter(cur.fetchall()), None) # returns 0 or 1 rows message_id = row['id'] if row is not None else None # If we've never stored the message, store it now. if message_id is None: cur = db.execute(""" INSERT INTO messages ( date, subject, clean_subject, in_reply_to, size, message_id_header, x_gm_msgid, x_gm_thrid, sender_addr, from_addr, reply_to_addr, to_addr, cc_addr, bcc_addr ) VALUES ( :date, :subject, :clean_subject, :in_reply_to, :size, :message_id_header, :x_gm_msgid, :x_gm_thrid, :sender_addr, :from_addr, :reply_to_addr, :to_addr, :cc_addr, :bcc_addr ) """, msg_data) message_id = cur.lastrowid # Store the Gmail labels (these can be different in # different folders; e.g. messages in the 'Sent' folder are # missing the u'\\Sent' label) db.executemany(""" INSERT INTO folder_message_gm_labels (folder_name, message_id, label) VALUES (?, ?, ?) """, ((folder_name, message_id, label) for label in data['X-GM-LABELS'])) # Mark the message as being in the current folder. db.execute(""" UPDATE folder_messages SET message_id = ? WHERE folder_name = ? AND imap_uid = ? """, (message_id, folder_name, uid)) # Construct threads (assuming gmail for now) db.execute(""" INSERT INTO threads (x_gm_thrid) SELECT DISTINCT x_gm_thrid FROM messages """) db.execute(""" INSERT INTO thread_messages (thread_id, message_id) SELECT threads.id, messages.id FROM threads, messages WHERE threads.x_gm_thrid = messages.x_gm_thrid """) # Construct folder_threads db.execute(""" INSERT INTO folder_threads (folder_name, thread_id) SELECT DISTINCT folder_messages.folder_name, thread_messages.thread_id FROM folder_messages LEFT JOIN thread_messages USING (message_id) """)
def headers_boolean_test(): eq_(False, bool(MimeHeaders())) eq_(True, bool(MimeHeaders([('A', 1)])))
def headers_to_string_test(): ok_(str(MimeHeaders([('A', 1)])))
def __init__(self, message): MimeHeaders.__init__(self, [(k, _try_decode(k, v)) for k, v in message.items()]) self._m = message