def create_imap_message(db_session, log, account, folder, msg): """ IMAP-specific message creation logic. This is the one function in this file that gets to take an account object instead of an account_id, because we need to relate the account to ImapUids for versioning to work, since it needs to look up the namespace. Returns ------- imapuid : inbox.models.tables.imap.ImapUid New db object, which links to new Message and Block objects through relationships. All new objects are uncommitted. """ msg_class = SpoolMessage if msg.created else Message new_msg = msg_class.create(account=account, mid=msg.uid, folder_name=folder.name, received_date=msg.internaldate, flags=msg.flags, body_string=msg.body) if new_msg: imapuid = ImapUid(account=account, folder=folder, msg_uid=msg.uid, message=new_msg) imapuid.update_imap_flags(msg.flags) new_msg.is_draft = imapuid.is_draft new_msg.is_read = imapuid.is_seen # NOTE: This might be a good place to add FolderItem entries for # non-Gmail backends. return imapuid
def add_new_imapuid(db_session, log, gmessage, folder_name, acc): """ Add ImapUid object for this GMessage if we don't already have one. Parameters ---------- message : GMessage Message to add ImapUid for. folder_name : str Which folder to add the ImapUid in. acc : GmailAccount Which account to associate the message with. (Not looking this up within this function is a db access optimization.) """ if not db_session.query(ImapUid.msg_uid).join(Folder).filter( Folder.name == folder_name, ImapUid.msg_uid == gmessage.uid).all(): message = db_session.query(Message).filter_by( g_msgid=gmessage.g_metadata.msgid).one() new_imapuid = ImapUid( account=acc, folder=Folder.find_or_create(db_session, acc, folder_name), msg_uid=gmessage.uid, message=message) new_imapuid.update_imap_flags(gmessage.flags, gmessage.labels) db_session.add(new_imapuid) db_session.commit() else: log.debug('skipping imapuid creation', uid=gmessage.uid)
def create_imap_message(db_session, account, folder, msg): """ IMAP-specific message creation logic. Returns ------- imapuid : inbox.models.backends.imap.ImapUid New db object, which links to new Message and Block objects through relationships. All new objects are uncommitted. """ new_message = Message.create_from_synced( account=account, mid=msg.uid, folder_name=folder.name, received_date=msg.internaldate, body_string=msg.body ) # Check to see if this is a copy of a message that was first created # by the Nylas API. If so, don't create a new object; just use the old one. existing_copy = reconcile_message(new_message, db_session) if existing_copy is not None: new_message = existing_copy imapuid = ImapUid(account=account, folder=folder, msg_uid=msg.uid, message=new_message) imapuid.update_flags(msg.flags) if msg.g_labels is not None: imapuid.update_labels(msg.g_labels) # Update the message's metadata with db_session.no_autoflush: is_draft = imapuid.is_draft and (folder.canonical_name == "drafts" or folder.canonical_name == "all") update_message_metadata(db_session, account, new_message, is_draft) update_contacts_from_message(db_session, new_message, account.namespace) return imapuid
def test_deleting_from_a_message_with_multiple_uids(db): # Now check that deleting a imapuid from a message with # multiple uids doesn't delete the message itself ACCOUNT_ID = 1 NAMESPACE_ID = 1 account = db.session.query(Account).get(ACCOUNT_ID) inbox_folder = account.inbox_folder sent_folder = account.sent_folder thread = add_fake_thread(db.session, NAMESPACE_ID,) message = add_fake_message(db.session, NAMESPACE_ID, thread) sent_uid = ImapUid(message=message, account=account, folder=sent_folder, msg_uid=1337) inbox_uid = ImapUid(message=message, account=account, folder=inbox_folder, msg_uid=2222) db.session.add(sent_uid) db.session.add(inbox_uid) db.session.commit() remove_messages(ACCOUNT_ID, db.session, [2222], inbox_folder.name) msg = db.session.query(Message).get(message.id) assert msg is not None, "the associated message should not have been deleted" assert len(msg.imapuids) == 1, "the message should have only one imapuid"
def create_imap_message(db_session, log, account, folder, msg): """ IMAP-specific message creation logic. This is the one function in this file that gets to take an account object instead of an account_id, because we need to relate the account to ImapUids for versioning to work, since it needs to look up the namespace. Returns ------- imapuid : inbox.models.tables.imap.ImapUid New db object, which links to new Message and Block objects through relationships. All new objects are uncommitted. """ new_msg = Message.create_from_synced(account=account, mid=msg.uid, folder_name=folder.name, received_date=msg.internaldate, body_string=msg.body) # Check to see if this is a copy of a message that was first created # by the Inbox API. If so, don't create a new object; just use the old one. existing_copy = reconcile_message(new_msg, db_session) if existing_copy is not None: new_msg = existing_copy imapuid = ImapUid(account=account, folder=folder, msg_uid=msg.uid, message=new_msg) imapuid.update_flags_and_labels(msg.flags, msg.g_labels) new_msg.is_draft = imapuid.is_draft new_msg.is_read = imapuid.is_seen update_contacts_from_message(db_session, new_msg, account.namespace) return imapuid
def resync_uids_impl(self): # NOTE: first, let's check if the UIVDALIDITY change was spurious, if # it is, just discard it and go on, if it isn't, drop the relevant # entries (filtering by account and folder IDs) from the imapuid table, # download messages, if necessary - in case a message has changed UID - # update UIDs, and discard orphaned messages. -siro with mailsync_session_scope() as db_session: folder_info = db_session.query(ImapFolderInfo). \ filter_by(account_id=self.account_id, folder_id=self.folder_id).one() cached_uidvalidity = folder_info.uidvalidity with self.conn_pool.get() as crispin_client: crispin_client.select_folder(self.folder_name, lambda *args: True) uidvalidity = crispin_client.selected_uidvalidity if uidvalidity <= cached_uidvalidity: log.debug('UIDVALIDITY unchanged') return invalid_uids = db_session.query(ImapUid). \ filter_by(account_id=self.account_id, folder_id=self.folder_id) data_sha256_message = {uid.message.data_sha256: uid.message for uid in invalid_uids} for uid in invalid_uids: db_session.delete(uid) # NOTE: this is necessary (and OK since it doesn't persist any # data) to maintain the order between UIDs deletion and # insertion. Without this, I was seeing constraints violation # on the imapuid table. -siro db_session.flush() remote_uids = crispin_client.all_uids() for remote_uid in remote_uids: raw_message = crispin_client.uids([remote_uid])[0] data_sha256 = sha256(raw_message.body).hexdigest() if data_sha256 in data_sha256_message: message = data_sha256_message[data_sha256] # Create a new imapuid uid = ImapUid(msg_uid=raw_message.uid, message_id=message.id, account_id=self.account_id, folder_id=self.folder_id) uid.update_flags(raw_message.flags) db_session.add(uid) # Update the existing message's metadata too common.update_message_metadata(db_session, uid) del data_sha256_message[data_sha256] else: self.download_and_commit_uids(crispin_client, [remote_uid]) self.heartbeat_status.publish() # FIXME: do we want to throttle the account when recovering # from UIDVALIDITY changes? -siro for message in data_sha256_message.itervalues(): db_session.delete(message) folder_info.uidvalidity = uidvalidity folder_info.highestmodseq = None
def resync_uids_impl(self): # NOTE: first, let's check if the UIVDALIDITY change was spurious, if # it is, just discard it and go on, if it isn't, drop the relevant # entries (filtering by account and folder IDs) from the imapuid table, # download messages, if necessary - in case a message has changed UID - # update UIDs, and discard orphaned messages. -siro with mailsync_session_scope() as db_session: folder_info = db_session.query(ImapFolderInfo). \ filter_by(account_id=self.account_id, folder_id=self.folder_id).one() cached_uidvalidity = folder_info.uidvalidity with self.conn_pool.get() as crispin_client: crispin_client.select_folder(self.folder_name, lambda *args: True) uidvalidity = crispin_client.selected_uidvalidity if uidvalidity <= cached_uidvalidity: log.debug('UIDVALIDITY unchanged') return invalid_uids = db_session.query(ImapUid). \ filter_by(account_id=self.account_id, folder_id=self.folder_id) data_sha256_message = { uid.message.data_sha256: uid.message for uid in invalid_uids } for uid in invalid_uids: db_session.delete(uid) # NOTE: this is necessary (and OK since it doesn't persist any # data) to maintain the order between UIDs deletion and # insertion. Without this, I was seeing constraints violation # on the imapuid table. -siro db_session.flush() remote_uids = crispin_client.all_uids() for remote_uid in remote_uids: raw_message = crispin_client.uids([remote_uid])[0] data_sha256 = sha256(raw_message.body).hexdigest() if data_sha256 in data_sha256_message: message = data_sha256_message[data_sha256] uid = ImapUid(msg_uid=raw_message.uid, message_id=message.id, account_id=self.account_id, folder_id=self.folder_id) uid.update_flags_and_labels(raw_message.flags, raw_message.g_labels) db_session.add(uid) del data_sha256_message[data_sha256] else: self.download_and_commit_uids(crispin_client, self.folder_name, [remote_uid]) self.heartbeat_status.publish() # FIXME: do we want to throttle the account when recovering # from UIDVALIDITY changes? -siro for message in data_sha256_message.itervalues(): db_session.delete(message) folder_info.uidvalidity = uidvalidity folder_info.highestmodseq = None
def __deduplicate_message_object_creation(self, db_session, raw_messages, account): """ We deduplicate messages based on g_msgid: if we've previously saved a Message object for this raw message, we don't create a new one. But we do create a new ImapUid, associate it to the message, and update flags and categories accordingly. Note: we could do this prior to downloading the actual message body, but that's really more complicated than it's worth. This operation is not super common unless you're regularly moving lots of messages to trash or spam, and even then the overhead of just downloading the body is generally not that high. """ new_g_msgids = {msg.g_msgid for msg in raw_messages} existing_g_msgids = g_msgids(self.namespace_id, db_session, in_=new_g_msgids) brand_new_messages = [m for m in raw_messages if m.g_msgid not in existing_g_msgids] previously_synced_messages = [m for m in raw_messages if m.g_msgid in existing_g_msgids] if previously_synced_messages: log.info('saving new uids for existing messages', count=len(previously_synced_messages)) account = Account.get(self.account_id, db_session) folder = Folder.get(self.folder_id, db_session) for raw_message in previously_synced_messages: message_obj = db_session.query(Message).filter( Message.namespace_id == self.namespace_id, Message.g_msgid == raw_message.g_msgid).first() if message_obj is None: log.warning( 'Message disappeared while saving new uid', g_msgid=raw_message.g_msgid, uid=raw_message.uid) brand_new_messages.append(raw_message) continue already_have_uid = ( (raw_message.uid, self.folder_id) in {(u.msg_uid, u.folder_id) for u in message_obj.imapuids} ) if already_have_uid: log.warning('Skipping existing UID for message', uid=raw_message.uid, message_id=message_obj.id) continue uid = ImapUid(account=account, folder=folder, msg_uid=raw_message.uid, message=message_obj) uid.update_flags(raw_message.flags) uid.update_labels(raw_message.g_labels) common.update_message_metadata( db_session, account, message_obj, uid.is_draft) db_session.commit() return brand_new_messages
def test_generic_grouping(db, default_account): thread = add_fake_thread(db.session, default_account.namespace.id) message = add_fake_message(db.session, default_account.namespace.id, thread, subject="Golden Gate Park next Sat") folder = Folder(account=default_account, name='Inbox', canonical_name='inbox') ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) thread = add_fake_thread(db.session, default_account.namespace.id) new_namespace = Namespace() db.session.add(new_namespace) db.session.commit() message = add_fake_message(db.session, new_namespace.id, thread, subject="Golden Gate Park next Sat") thread = fetch_corresponding_thread(db.session, default_account.namespace.id, message) assert thread is None, ("fetch_similar_threads should " "heed namespace boundaries")
def test_truncate_imapuid_extra_flags(db, default_account, message, folder): imapuid = ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) imapuid.update_flags(['We', 'the', 'People', 'of', 'the', 'United', 'States', 'in', 'Order', 'to', 'form', 'a', 'more', 'perfect', 'Union', 'establish', 'Justice', 'insure', 'domestic', 'Tranquility', 'provide', 'for', 'the', 'common', 'defence', 'promote', 'the', 'general', 'Welfare', 'and', 'secure', 'the', 'Blessings', 'of', 'Liberty', 'to', 'ourselves', 'and', 'our', 'Posterity', 'do', 'ordain', 'and', 'establish', 'this', 'Constitution', 'for', 'the', 'United', 'States', 'of', 'America']) assert len(json.dumps(imapuid.extra_flags)) < 255
def test_generic_flags_refresh_expunges_transient_uids(db, generic_account, inbox_folder, mock_imapclient, monkeypatch): # Check that we delete UIDs which are synced but quickly deleted, so never # show up in flags refresh. uid_dict = uids.example() mock_imapclient.add_folder_data(inbox_folder.name, uid_dict) inbox_folder.imapfolderinfo = ImapFolderInfo(account=generic_account, uidvalidity=1, uidnext=1) db.session.commit() folder_sync_engine = FolderSyncEngine(generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1)) folder_sync_engine.initial_sync() # Don't sleep at the end of poll_impl before returning. folder_sync_engine.poll_frequency = 0 folder_sync_engine.poll_impl() msg = db.session.query(Message).filter_by( namespace_id=generic_account.namespace.id).first() transient_uid = ImapUid(folder=inbox_folder, account=generic_account, message=msg, msg_uid=max(uid_dict) + 1) db.session.add(transient_uid) db.session.commit() folder_sync_engine.last_slow_refresh = None folder_sync_engine.poll_impl() with pytest.raises(ObjectDeletedError): transient_uid.id
def add_new_imapuids(crispin_client, remote_g_metadata, syncmanager_lock, uids): """ Add ImapUid entries only for (already-downloaded) messages. If a message has already been downloaded via another folder, we only need to add `ImapUid` accounting for the current folder. `Message` objects etc. have already been created. """ flags = crispin_client.flags(uids) with syncmanager_lock: with mailsync_session_scope() as db_session: # Since we prioritize download for messages in certain threads, we # may already have ImapUid entries despite calling this method. local_folder_uids = {uid for uid, in db_session.query(ImapUid.msg_uid).join(Folder) .filter( ImapUid.account_id == crispin_client.account_id, Folder.name == crispin_client.selected_folder_name, ImapUid.msg_uid.in_(uids))} uids = [uid for uid in uids if uid not in local_folder_uids] if uids: acc = db_session.query(GmailAccount).get( crispin_client.account_id) # collate message objects to relate the new imapuids to imapuid_for = dict([(metadata.msgid, uid) for (uid, metadata) in remote_g_metadata.items() if uid in uids]) imapuid_g_msgids = [remote_g_metadata[uid].msgid for uid in uids] message_for = dict([(imapuid_for[m.g_msgid], m) for m in db_session.query(Message).join(ImapThread) .filter( Message.g_msgid.in_(imapuid_g_msgids), ImapThread.namespace_id == acc.namespace.id)]) # Stop Folder.find_or_create()'s query from triggering a flush. with db_session.no_autoflush: new_imapuids = [ImapUid( account=acc, folder=Folder.find_or_create( db_session, acc, crispin_client.selected_folder_name), msg_uid=uid, message=message_for[uid]) for uid in uids if uid in message_for] for item in new_imapuids: # skip uids which have disappeared in the meantime if item.msg_uid in flags: item.update_flags_and_labels( flags[item.msg_uid].flags, flags[item.msg_uid].labels) db_session.add_all(new_imapuids) db_session.commit()
def test_generic_grouping(db, default_account): thread = add_fake_thread(db.session, default_account.namespace.id) message = add_fake_message( db.session, default_account.namespace.id, thread, subject="Golden Gate Park next Sat", ) folder = Folder(account=default_account, name="Inbox", canonical_name="inbox") ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) thread = add_fake_thread(db.session, default_account.namespace.id) account = add_generic_imap_account(db.session) message = add_fake_message(db.session, account.namespace.id, thread, subject="Golden Gate Park next Sat") thread = fetch_corresponding_thread(db.session, default_account.namespace.id, message) assert thread is None, "fetch_similar_threads should " "heed namespace boundaries"
def test_truncate_imapuid_extra_flags(db, default_account, message, folder): imapuid = ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) imapuid.update_flags([ 'We', 'the', 'People', 'of', 'the', 'United', 'States', 'in', 'Order', 'to', 'form', 'a', 'more', 'perfect', 'Union', 'establish', 'Justice', 'insure', 'domestic', 'Tranquility', 'provide', 'for', 'the', 'common', 'defence', 'promote', 'the', 'general', 'Welfare', 'and', 'secure', 'the', 'Blessings', 'of', 'Liberty', 'to', 'ourselves', 'and', 'our', 'Posterity', 'do', 'ordain', 'and', 'establish', 'this', 'Constitution', 'for', 'the', 'United', 'States', 'of', 'America' ]) assert len(json.dumps(imapuid.extra_flags)) < 255
def add_new_imapuid(db_session, gmessage, folder_name, acc): """ Add ImapUid object for this GMessage if we don't already have one. Parameters ---------- message : GMessage Message to add ImapUid for. folder_name : str Which folder to add the ImapUid in. acc : GmailAccount Which account to associate the message with. (Not looking this up within this function is a db access optimization.) """ if not db_session.query(ImapUid.msg_uid).join(Folder).filter( Folder.name == folder_name, ImapUid.account_id == acc.id, ImapUid.msg_uid == gmessage.uid).all(): try: message = db_session.query(Message).join(ImapThread).filter( ImapThread.g_thrid == gmessage.g_metadata.thrid, Message.g_thrid == gmessage.g_metadata.thrid, Message.g_msgid == gmessage.g_metadata.msgid, ImapThread.namespace_id == acc.namespace.id).one() except NoResultFound: # this may occur when a thread is expanded and those messages are # downloaded and committed, then new messages on that thread arrive # and get added to the download queue before this code is run log.debug('no Message object found, skipping imapuid creation', uid=gmessage.uid, g_msgid=gmessage.g_metadata.msgid) return new_imapuid = ImapUid(account=acc, folder=Folder.find_or_create( db_session, acc, folder_name), msg_uid=gmessage.uid, message=message) new_imapuid.update_imap_flags(gmessage.flags, gmessage.labels) new_imapuid.g_labels = [label for label in gmessage.labels] db_session.add(new_imapuid) db_session.commit() else: log.debug('skipping imapuid creation', uid=gmessage.uid, g_msgid=gmessage.g_metadata.msgid)
def add_fake_imapuid(db_session, account_id, message, folder, msg_uid): from inbox.models.backends.imap import ImapUid imapuid = ImapUid(account_id=account_id, message=message, folder=folder, msg_uid=msg_uid) db_session.add(imapuid) db_session.commit() return imapuid
def test_soft_delete(db, config): from inbox.models import Folder, Message from inbox.models.backends.imap import ImapUid f = Folder(name='DOES NOT EXIST', account_id=ACCOUNT_ID) db.session.add(f) db.session.flush() m = Message() m.thread_id = 1 m.received_date = datetime.datetime.utcnow() m.size = 0 m.sanitized_body = "" m.snippet = "" u = ImapUid(message=m, account_id=ACCOUNT_ID, folder_id=f.id, msg_uid=9999, extra_flags="") db.session.add_all([m, u]) f.mark_deleted() u.mark_deleted() db.session.commit() m_id = m.id # bypass custom query method to confirm creation db.new_session(ignore_soft_deletes=False) f = db.session.query(Folder).filter_by(name='DOES NOT EXIST').one() assert f, "Can't find Folder object" assert f.deleted_at is not None, "Folder not marked as deleted" db.new_session(ignore_soft_deletes=True) with pytest.raises(NoResultFound): folders = db.session.query(Folder).filter( Folder.name == 'DOES NOT EXIST').one() count = db.session.query(Folder).filter( Folder.name == 'DOES NOT EXIST').count() assert count == 0, "Shouldn't find any deleted folders!" m = db.session.query(Message).filter_by(id=m_id).one() assert not m.imapuids, "imapuid was deleted!"
def add_new_imapuid(db_session, gmessage, folder_name, acc): """ Add ImapUid object for this GMessage if we don't already have one. Parameters ---------- message : GMessage Message to add ImapUid for. folder_name : str Which folder to add the ImapUid in. acc : GmailAccount Which account to associate the message with. (Not looking this up within this function is a db access optimization.) """ if not db_session.query(ImapUid.msg_uid).join(Folder).filter( Folder.name == folder_name, ImapUid.account_id == acc.id, ImapUid.msg_uid == gmessage.uid).all(): try: message = db_session.query(Message).join(ImapThread).filter( ImapThread.g_thrid == gmessage.g_metadata.thrid, Message.g_thrid == gmessage.g_metadata.thrid, Message.g_msgid == gmessage.g_metadata.msgid, ImapThread.namespace_id == acc.namespace.id).one() except NoResultFound: # this may occur when a thread is expanded and those messages are # downloaded and committed, then new messages on that thread arrive # and get added to the download queue before this code is run log.debug('no Message object found, skipping imapuid creation', uid=gmessage.uid, g_msgid=gmessage.g_metadata.msgid) return new_imapuid = ImapUid( account=acc, folder=Folder.find_or_create(db_session, acc, folder_name), msg_uid=gmessage.uid, message=message) new_imapuid.update_imap_flags(gmessage.flags, gmessage.labels) new_imapuid.g_labels = [label for label in gmessage.labels] db_session.add(new_imapuid) db_session.commit() else: log.debug('skipping imapuid creation', uid=gmessage.uid, g_msgid=gmessage.g_metadata.msgid)
def test_generic_foldersyncengine(db, folder_sync_engine): thread = add_fake_thread(db.session, NAMESPACE_ID) message = add_fake_message(db.session, NAMESPACE_ID, thread, subject="Golden Gate Park next Sat") imapuid = ImapUid(message=message, account_id=ACCOUNT_ID, msg_uid=2222) messages = folder_sync_engine.fetch_similar_threads(db.session, imapuid) assert messages == [], ("fetch_similar_threads should " "heed namespace boundaries")
def test_update_metadata(db): """Check that threads are updated correctly when a label that we haven't seen before is added to multiple threads -- previously, this would with an IntegrityError because autoflush was disabled.""" first_thread = db.session.query(Thread).get(1) second_thread = db.session.query(Thread).get(2) folder = db.session.query(Folder).filter( Folder.account_id == ACCOUNT_ID, Folder.name == '[Gmail]/All Mail').one() uids = [] first_thread_uids = (22222, 22223) for msg_uid in first_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, first_thread) uids.append( ImapUid(account_id=ACCOUNT_ID, message=message, msg_uid=msg_uid, folder=folder)) second_thread_uids = (22224, 22226) for msg_uid in second_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, second_thread) uids.append( ImapUid(account_id=ACCOUNT_ID, message=message, msg_uid=msg_uid, folder=folder)) db.session.add_all(uids) db.session.commit() msg_uids = first_thread_uids + second_thread_uids new_flags = { msg_uid: GmailFlags((), (u'\\some_new_label', )) for msg_uid in msg_uids } update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, msg_uids, new_flags) db.session.commit() assert 'some_new_label' in [tag.name for tag in first_thread.tags] assert 'some_new_label' in [tag.name for tag in second_thread.tags]
def test_adding_message_to_thread(db): """recompute_thread_labels is not invoked when a new message is added (only when UID metadata changes, or when a UID is deleted). Test that tag changes work when adding messages to a thread.""" account = db.session.query(Account).get(ACCOUNT_ID) account.namespace.create_canonical_tags() thread = db.session.query(Thread).get(THREAD_ID) account.trash_folder = Folder(name='Trash', account=account) FolderItem(thread=thread, folder=account.trash_folder) folder_names = [folder.name for folder in thread.folders] m = Message(namespace_id=account.namespace.id, subject='test message', thread_id=thread.id, received_date=datetime.datetime.now(), size=64, body="body", snippet="snippet") uid = ImapUid(account=account, message=m, g_labels=['\\Inbox', 'test-label'], msg_uid=22L, folder_id=account.inbox_folder.id) uid.folder = account.inbox_folder uid2 = ImapUid(account=account, message=m, g_labels=['test-2'], msg_uid=24L, folder_id=account.trash_folder.id) uid2.folder = account.trash_folder thread.messages.append(m) add_any_new_thread_labels(thread, uid, db.session) add_any_new_thread_labels(thread, uid2, db.session) folder_names = [folder.name for folder in thread.folders] for folder in folder_names: assert folder in ['Inbox', 'Trash', 'test-label', 'test-2', '[Gmail]/All Mail', '[Gmail]/Important'],\ "all folders should be present" # Now, remove the message m.imapuids.remove(uid2) db.session.delete(uid2) db.session.flush() recompute_thread_labels(thread, db.session) folder_names = [folder.name for folder in thread.folders] assert 'test-2' not in folder_names,\ "test-2 label should have been removed from thread"
def test_deleting_from_a_message_with_multiple_uids(db): # Now check that deleting a imapuid from a message with # multiple uids doesn't delete the message itself ACCOUNT_ID = 1 UID = 380 account = db.session.query(Account).get(ACCOUNT_ID) inbox_folder = account.inbox_folder spam_folder = account.spam_folder old_number_of_imapuids = len(inbox_folder.imapuids) imapuid = db.session.query(ImapUid).filter( ImapUid.account_id == ACCOUNT_ID, ImapUid.msg_uid == UID).one() message = imapuid.message message_id = message.id uid = ImapUid() uid.message = message uid.account = account uid.folder = spam_folder uid.msg_uid = 1337 db.session.commit() remove_messages(ACCOUNT_ID, db.session, [UID], inbox_folder.name) msg = db.session.query(Message).get(message_id) assert msg is not None, "the associated message should not have been deleted" assert len(msg.imapuids) == 1, "the message should have only one imapuid"
def test_soft_delete(db, config): from inbox.models import Folder, Message from inbox.models.backends.imap import ImapUid f = Folder(name='DOES NOT EXIST', account_id=ACCOUNT_ID) db.session.add(f) db.session.flush() m = Message() m.namespace_id = NAMESPACE_ID m.thread_id = 1 m.received_date = datetime.datetime.utcnow() m.size = 0 m.sanitized_body = "" m.snippet = "" u = ImapUid(message=m, account_id=ACCOUNT_ID, folder_id=f.id, msg_uid=9999, extra_flags="") db.session.add_all([m, u]) f.mark_deleted() u.mark_deleted() db.session.commit() m_id = m.id # bypass custom query method to confirm creation db.new_session(ignore_soft_deletes=False) f = db.session.query(Folder).filter_by(name='DOES NOT EXIST').one() assert f, "Can't find Folder object" assert f.deleted_at is not None, "Folder not marked as deleted" db.new_session(ignore_soft_deletes=True) with pytest.raises(NoResultFound): db.session.query(Folder).filter(Folder.name == 'DOES NOT EXIST').one() count = db.session.query(Folder).filter( Folder.name == 'DOES NOT EXIST').count() assert count == 0, "Shouldn't find any deleted folders!" m = db.session.query(Message).filter_by(id=m_id).one() assert not m.imapuids, "imapuid was deleted!"
def __deduplicate_message_object_creation(self, db_session, raw_messages, account): """ We deduplicate messages based on g_msgid: if we've previously saved a Message object for this raw message, we don't create a new one. But we do create a new ImapUid, associate it to the message, and update flags and categories accordingly. Note: we could do this prior to downloading the actual message body, but that's really more complicated than it's worth. This operation is not super common unless you're regularly moving lots of messages to trash or spam, and even then the overhead of just downloading the body is generally not that high. """ new_g_msgids = {msg.g_msgid for msg in raw_messages} existing_g_msgids = g_msgids(self.namespace_id, db_session, in_=new_g_msgids) brand_new_messages = [ m for m in raw_messages if m.g_msgid not in existing_g_msgids ] previously_synced_messages = [ m for m in raw_messages if m.g_msgid in existing_g_msgids ] if previously_synced_messages: log.info('saving new uids for existing messages', count=len(previously_synced_messages)) account = Account.get(self.account_id, db_session) folder = Folder.get(self.folder_id, db_session) for raw_message in previously_synced_messages: message_obj = db_session.query(Message).filter( Message.namespace_id == self.namespace_id, Message.g_msgid == raw_message.g_msgid).first() if message_obj is None: log.warning('Message disappeared while saving new uid', g_msgid=raw_message.g_msgid, uid=raw_message.uid) brand_new_messages.append(raw_message) continue already_have_uid = ((raw_message.uid, self.folder_id) in {(u.msg_uid, u.folder_id) for u in message_obj.imapuids}) if already_have_uid: log.warning('Skipping existing UID for message', uid=raw_message.uid, message_id=message_obj.id) continue uid = ImapUid(account=account, folder=folder, msg_uid=raw_message.uid, message=message_obj) uid.update_flags(raw_message.flags) uid.update_labels(raw_message.g_labels) common.update_message_metadata(db_session, account, message_obj, uid.is_draft) db_session.commit() return brand_new_messages
def test_generic_grouping(db, generic_account): thread = add_fake_thread(db.session, NAMESPACE_ID) message = add_fake_message(db.session, NAMESPACE_ID, thread, subject="Golden Gate Park next Sat") imapuid = ImapUid(message=message, account_id=ACCOUNT_ID, msg_uid=2222) thread = add_fake_thread(db.session, generic_account.namespace.id) message = add_fake_message(db.session, NAMESPACE_ID + 1, thread, subject="Golden Gate Park next Sat") thread = fetch_corresponding_thread(db.session, generic_account.namespace.id, message) assert thread is None, ("fetch_similar_threads should " "heed namespace boundaries")
def create_imap_message(db_session, log, account, folder, msg): """ IMAP-specific message creation logic. This is the one function in this file that gets to take an account object instead of an account_id, because we need to relate the account to ImapUids for versioning to work, since it needs to look up the namespace. Returns ------- imapuid : inbox.models.tables.imap.ImapUid New db object, which links to new Message and Block objects through relationships. All new objects are uncommitted. """ new_msg = Message(account=account, mid=msg.uid, folder_name=folder.name, received_date=msg.internaldate, flags=msg.flags, body_string=msg.body) imapuid = ImapUid(account=account, folder=folder, msg_uid=msg.uid, message=new_msg) imapuid.update_imap_flags(msg.flags, msg.g_labels) new_msg.is_draft = imapuid.is_draft new_msg.is_read = imapuid.is_seen update_contacts_from_message(db_session, new_msg, account.id) # NOTE: This might be a good place to add FolderItem entries for # non-Gmail backends. return imapuid
def test_adding_message_to_thread(db): """recompute_thread_labels is not invoked when a new message is added (only when UID metadata changes, or when a UID is deleted). Test that tag changes work when adding messages to a thread.""" account = db.session.query(Account).get(ACCOUNT_ID) account.namespace.create_canonical_tags() thread = db.session.query(Thread).get(THREAD_ID) account.trash_folder = Folder(name='Trash', account=account) fld_item = FolderItem(thread=thread, folder=account.trash_folder) folder_names = [folder.name for folder in thread.folders] m = Message(namespace_id=account.namespace.id, subject='test message', thread_id=thread.id, received_date=datetime.datetime.now(), size=64, sanitized_body="body", snippet="snippet") uid = ImapUid(account=account, message=m, g_labels=['\\Inbox', 'test-label'], msg_uid=22L, folder_id=account.inbox_folder.id) uid.folder = account.inbox_folder uid2 = ImapUid(account=account, message=m, g_labels=['test-2'], msg_uid=24L, folder_id=account.trash_folder.id) uid2.folder = account.trash_folder thread.messages.append(m) add_any_new_thread_labels(thread, uid, db.session) add_any_new_thread_labels(thread, uid2, db.session) folder_names = [folder.name for folder in thread.folders] for folder in folder_names: assert folder in ['Inbox', 'Trash', 'test-label', 'test-2', '[Gmail]/All Mail', '[Gmail]/Important'],\ "all folders should be present" # Now, remove the message m.imapuids.remove(uid2) db.session.delete(uid2) db.session.flush() recompute_thread_labels(thread, db.session) folder_names = [folder.name for folder in thread.folders] assert 'test-2' not in folder_names,\ "test-2 label should have been removed from thread"
def test_threading_limit(db, folder_sync_engine, monkeypatch): """Test that custom threading doesn't produce arbitrarily long threads, which eventually break things.""" from inbox.models import Message, Thread, Account # Shorten bound to make test faster MAX_THREAD_LENGTH = 10 monkeypatch.setattr( 'inbox.mailsync.backends.imap.generic.MAX_THREAD_LENGTH', MAX_THREAD_LENGTH) namespace_id = folder_sync_engine.namespace_id account = db.session.query(Account).get(folder_sync_engine.account_id) account.namespace.create_canonical_tags() account.inbox_folder = Folder(account=account, name='Inbox', canonical_name='inbox') folder = account.inbox_folder msg = MockRawMessage([]) for i in range(3 * MAX_THREAD_LENGTH): m = Message() m.namespace_id = namespace_id m.received_date = datetime.datetime.utcnow() m.references = [] m.size = 0 m.body = '' m.from_addr = [("Karim Hamidou", "*****@*****.**")] m.to_addr = [("Eben Freeman", "*****@*****.**")] m.snippet = '' m.subject = 'unique subject' uid = ImapUid(message=m, account=account, msg_uid=2222 + i, folder=folder) folder_sync_engine.add_message_attrs(db.session, uid, msg) db.session.add(m) db.session.commit() new_threads = db.session.query(Thread). \ filter(Thread.subject == 'unique subject').all() assert len(new_threads) == 3 assert all( len(thread.messages) == MAX_THREAD_LENGTH for thread in new_threads)
def test_gmail_label_sync(db): folder = db.session.query(Folder).filter( Folder.account_id == ACCOUNT_ID, Folder.name == '[Gmail]/All Mail').one() account = db.session.query(Account).get(ACCOUNT_ID) if account.important_folder is not None: db.session.delete(account.important_folder) thread = db.session.query(Thread).get(1) message = add_fake_message(db.session, NAMESPACE_ID, thread) db.session.add( ImapUid(account_id=ACCOUNT_ID, message=message, msg_uid=22222, folder=folder)) db.session.commit() new_flags = {22222: GmailFlags((), (u'\\Important', u'\\Starred', u'foo'))} update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [22222], new_flags) thread_tag_names = {tag.name for tag in thread.tags} assert {'important', 'starred', 'foo'}.issubset(thread_tag_names)
def create_imap_message(db_session, account, folder, msg): """ IMAP-specific message creation logic. Returns ------- imapuid : inbox.models.backends.imap.ImapUid New db object, which links to new Message and Block objects through relationships. All new objects are uncommitted. """ log.debug("creating message", account_id=account.id, folder_name=folder.name, mid=msg.uid) new_message = Message.create_from_synced( account=account, mid=msg.uid, folder_name=folder.name, received_date=msg.internaldate, body_string=msg.body, ) # Check to see if this is a copy of a message that was first created # by the Nylas API. If so, don't create a new object; just use the old one. existing_copy = reconcile_message(new_message, db_session) if existing_copy is not None: new_message = existing_copy imapuid = ImapUid(account=account, folder=folder, msg_uid=msg.uid, message=new_message) imapuid.update_flags(msg.flags) if msg.g_labels is not None: imapuid.update_labels(msg.g_labels) # Update the message's metadata with db_session.no_autoflush: is_draft = imapuid.is_draft and (folder.canonical_name == "drafts" or folder.canonical_name == "all") update_message_metadata(db_session, account, new_message, is_draft) update_contacts_from_message(db_session, new_message, account.namespace.id) return imapuid
def test_truncate_imapuid_extra_flags(db, default_account, message, folder): imapuid = ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) imapuid.update_flags( [ "We", "the", "People", "of", "the", "United", "States", "in", "Order", "to", "form", "a", "more", "perfect", "Union", "establish", "Justice", "insure", "domestic", "Tranquility", "provide", "for", "the", "common", "defence", "promote", "the", "general", "Welfare", "and", "secure", "the", "Blessings", "of", "Liberty", "to", "ourselves", "and", "our", "Posterity", "do", "ordain", "and", "establish", "this", "Constitution", "for", "the", "United", "States", "of", "America", ] ) assert len(json.dumps(imapuid.extra_flags)) < 255
def test_truncate_imapuid_extra_flags(db, default_account, message, folder): imapuid = ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) imapuid.update_flags([ "We", "the", "People", "of", "the", "United", "States", "in", "Order", "to", "form", "a", "more", "perfect", "Union", "establish", "Justice", "insure", "domestic", "Tranquility", "provide", "for", "the", "common", "defence", "promote", "the", "general", "Welfare", "and", "secure", "the", "Blessings", "of", "Liberty", "to", "ourselves", "and", "our", "Posterity", "do", "ordain", "and", "establish", "this", "Constitution", "for", "the", "United", "States", "of", "America", ]) assert len(json.dumps(imapuid.extra_flags)) < 255