def test_update_metadata(db): """Check that threads are updated correctly when a label that we haven't seen before is added to multiple threads -- previously, this would with an IntegrityError because autoflush was disabled.""" first_thread = db.session.query(Thread).get(1) second_thread = db.session.query(Thread).get(2) folder = db.session.query(Folder).filter( Folder.account_id == ACCOUNT_ID, Folder.name == '[Gmail]/All Mail').one() uids = [] first_thread_uids = (22222, 22223) for msg_uid in first_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, first_thread) uids.append(ImapUid(account_id=ACCOUNT_ID, message=message, msg_uid=msg_uid, folder=folder)) second_thread_uids = (22224, 22226) for msg_uid in second_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, second_thread) uids.append(ImapUid(account_id=ACCOUNT_ID, message=message, msg_uid=msg_uid, folder=folder)) db.session.add_all(uids) db.session.commit() msg_uids = first_thread_uids + second_thread_uids new_flags = {msg_uid: GmailFlags((), (u'\\some_new_label',)) for msg_uid in msg_uids} update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, msg_uids, new_flags) db.session.commit() assert 'some_new_label' in [tag.name for tag in first_thread.tags] assert 'some_new_label' in [tag.name for tag in second_thread.tags]
def _run_impl(self): self.log.info('Starting LabelRenameHandler', label_name=self.label_name) with connection_pool(self.account_id).get() as crispin_client: folder_names = [] with session_scope(self.account_id) as db_session: folders = db_session.query(Folder).filter( Folder.account_id == self.account_id) folder_names = [folder.name for folder in folders] db_session.expunge_all() for folder_name in folder_names: crispin_client.select_folder(folder_name, uidvalidity_cb) found_uids = crispin_client.search_uids( ['X-GM-LABELS', utf7_encode(self.label_name)]) flags = crispin_client.flags(found_uids) self.log.info('Running metadata update for folder', folder_name=folder_name) with session_scope(self.account_id) as db_session: common.update_metadata(self.account_id, folder.id, flags, db_session) db_session.commit()
def refresh_flags_impl(self, crispin_client, max_uids): crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) with session_scope(self.namespace_id) as db_session: local_uids = common.local_uids(account_id=self.account_id, session=db_session, folder_id=self.folder_id, limit=max_uids) flags = crispin_client.flags(local_uids) if (max_uids in self.flags_fetch_results and self.flags_fetch_results[max_uids] == (local_uids, flags)): # If the flags fetch response is exactly the same as the last one # we got, then we don't need to persist any changes. log.debug('Unchanged flags refresh response, ' 'not persisting changes', max_uids=max_uids) return log.debug('Changed flags refresh response, persisting changes', max_uids=max_uids) expunged_uids = set(local_uids).difference(flags.keys()) common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids) with session_scope(self.namespace_id) as db_session: common.update_metadata(self.account_id, self.folder_id, self.folder_role, flags, db_session) self.flags_fetch_results[max_uids] = (local_uids, flags)
def test_renamed_label_refresh(db, default_account, thread, message, imapuid, folder, mock_imapclient, monkeypatch): # Check that imapuids see their labels refreshed after running # the LabelRenameHandler. msg_uid = imapuid.msg_uid uid_dict = {msg_uid: GmailFlags((), ('stale label',))} update_metadata(default_account.id, folder.id, uid_dict, db.session) new_flags = {msg_uid: {'FLAGS': ('\\Seen',), 'X-GM-LABELS': ('new label',)}} mock_imapclient._data['[Gmail]/All mail'] = new_flags mock_imapclient.add_folder_data(folder.name, new_flags) monkeypatch.setattr(MockIMAPClient, 'search', lambda x, y: [msg_uid]) rename_handler = LabelRenameHandler(default_account.id, default_account.namespace.id, 'new label') rename_handler.start() rename_handler.join() labels = list(imapuid.labels) assert len(labels) == 1 assert labels[0].name == 'new label'
def condstore_refresh_flags(self, crispin_client): new_highestmodseq = crispin_client.conn.folder_status( self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ'] # Ensure that we have an initial highestmodseq value stored before we # begin polling for changes. if self.highestmodseq is None: self.highestmodseq = new_highestmodseq if new_highestmodseq == self.highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < self.highestmodseq: # This should really never happen, but if it does, handle it. log.warning('got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=self.highestmodseq) return # Highestmodseq has changed, update accordingly. crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) changed_flags = crispin_client.condstore_changed_flags( self.highestmodseq) remote_uids = crispin_client.all_uids() with session_scope() as db_session: common.update_metadata(self.account_id, self.folder_id, changed_flags, db_session) local_uids = common.local_uids(self.account_id, db_session, self.folder_id) expunged_uids = set(local_uids).difference(remote_uids) common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids, db_session) db_session.commit() self.highestmodseq = new_highestmodseq
def test_update_metadata(db, folder): """Check that threads are updated correctly when a label that we haven't seen before is added to multiple threads -- previously, this would fail with an IntegrityError because autoflush was disabled.""" first_thread = add_fake_thread(db.session, NAMESPACE_ID) second_thread = add_fake_thread(db.session, NAMESPACE_ID) uids = [] first_thread_uids = (22222, 22223) for msg_uid in first_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, first_thread) uids.append(add_fake_imapuid(db.session, ACCOUNT_ID, message, folder, msg_uid)) second_thread_uids = (22224, 22226) for msg_uid in second_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, second_thread) uids.append(add_fake_imapuid(db.session, ACCOUNT_ID, message, folder, msg_uid)) db.session.add_all(uids) db.session.commit() msg_uids = first_thread_uids + second_thread_uids new_flags = {msg_uid: GmailFlags((), (u"\\some_new_label",)) for msg_uid in msg_uids} update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, msg_uids, new_flags) db.session.commit() assert "some_new_label" in [tag.name for tag in first_thread.tags] assert "some_new_label" in [tag.name for tag in second_thread.tags]
def test_update_metadata(db, folder): """Check that threads are updated correctly when a label that we haven't seen before is added to multiple threads -- previously, this would fail with an IntegrityError because autoflush was disabled.""" first_thread = add_fake_thread(db.session, NAMESPACE_ID) second_thread = add_fake_thread(db.session, NAMESPACE_ID) uids = [] first_thread_uids = (22222, 22223) for msg_uid in first_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, first_thread) uids.append( add_fake_imapuid(db.session, ACCOUNT_ID, message, folder, msg_uid)) second_thread_uids = (22224, 22226) for msg_uid in second_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, second_thread) uids.append( add_fake_imapuid(db.session, ACCOUNT_ID, message, folder, msg_uid)) db.session.add_all(uids) db.session.commit() msg_uids = first_thread_uids + second_thread_uids new_flags = { msg_uid: GmailFlags((), (u'\\some_new_label', )) for msg_uid in msg_uids } update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, msg_uids, new_flags) db.session.commit() assert 'some_new_label' in [tag.name for tag in first_thread.tags] assert 'some_new_label' in [tag.name for tag in second_thread.tags]
def test_gmail_drafts_flag_constrained_by_folder( db, default_account, message, imapuid, folder ): new_flags = {imapuid.msg_uid: GmailFlags((), (u"\\Draft",), None)} update_metadata(default_account.id, folder.id, "all", new_flags, db.session) assert message.is_draft update_metadata(default_account.id, folder.id, "trash", new_flags, db.session) assert not message.is_draft
def test_gmail_drafts_flag_constrained_by_folder(db, default_account, message, imapuid, folder): new_flags = {imapuid.msg_uid: GmailFlags((), (u'\\Draft',), None)} update_metadata(default_account.id, folder.id, 'all', new_flags, db.session) assert message.is_draft update_metadata(default_account.id, folder.id, 'trash', new_flags, db.session) assert not message.is_draft
def test_gmail_label_sync(db, default_account, message, folder, imapuid, default_namespace): # Note that IMAPClient parses numeric labels into integer types. We have to # correctly handle those too. new_flags = {imapuid.msg_uid: GmailFlags((), (u"\\Important", u"\\Starred", u"foo", 42), None)} update_metadata(default_namespace.account.id, folder.id, folder.canonical_name, new_flags, db.session) category_canonical_names = {c.name for c in message.categories} category_display_names = {c.display_name for c in message.categories} assert "important" in category_canonical_names assert {"foo", "42"}.issubset(category_display_names)
def test_messages_deleted_asynchronously(db, default_account, thread, message, imapuid, folder): msg_uid = imapuid.msg_uid update_metadata(default_account.id, db.session, folder.name, folder.id, [msg_uid], {msg_uid: GmailFlags((), ('label',))}) assert 'label' in [cat.display_name for cat in message.categories] remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id) assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2 # Check that message categories do get updated synchronously. assert 'label' not in [cat.display_name for cat in message.categories]
def test_generic_drafts_flag_constrained_by_folder(db, generic_account, folder_role): msg_uid = 22 thread = add_fake_thread(db.session, generic_account.namespace.id) message = add_fake_message(db.session, generic_account.namespace.id, thread) folder = add_fake_folder(db.session, generic_account) add_fake_imapuid(db.session, generic_account.id, message, folder, msg_uid) new_flags = {msg_uid: Flags(("\\Draft",), None)} update_metadata(generic_account.id, folder.id, folder_role, new_flags, db.session) assert message.is_draft == (folder_role == "drafts")
def test_messages_deleted_asynchronously(db, default_account, thread, message, imapuid, folder): msg_uid = imapuid.msg_uid update_metadata(default_account.id, folder.id, {msg_uid: GmailFlags((), ('label', ))}, db.session) assert 'label' in [cat.display_name for cat in message.categories] remove_deleted_uids(default_account.id, folder.id, [msg_uid], db.session) assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2 # Check that message categories do get updated synchronously. assert 'label' not in [cat.display_name for cat in message.categories]
def test_generic_drafts_flag_constrained_by_folder(db, generic_account, folder_role): msg_uid = 22 thread = add_fake_thread(db.session, generic_account.namespace.id) message = add_fake_message(db.session, generic_account.namespace.id, thread) folder = add_fake_folder(db.session, generic_account) add_fake_imapuid(db.session, generic_account.id, message, folder, msg_uid) new_flags = {msg_uid: Flags((b"\\Draft",), None)} update_metadata(generic_account.id, folder.id, folder_role, new_flags, db.session) assert message.is_draft == (folder_role == "drafts")
def test_only_uids_deleted_synchronously(db, default_account, default_namespace, thread, message, imapuid, folder): msg_uid = imapuid.msg_uid update_metadata(default_account.id, db.session, folder.name, folder.id, [msg_uid], {msg_uid: GmailFlags((), ('label', ))}) assert 'label' in [t.name for t in thread.tags] remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id) assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 1 # Check that thread tags do get updated synchronously. assert 'label' not in [t.name for t in thread.tags]
def test_only_uids_deleted_synchronously(db, default_account, default_namespace, thread, message, imapuid, folder): msg_uid = imapuid.msg_uid update_metadata(default_account.id, db.session, folder.name, folder.id, [msg_uid], {msg_uid: GmailFlags((), ('label',))}) assert 'label' in [t.name for t in thread.tags] remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id) assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2 # Check that thread tags do get updated synchronously. assert 'label' not in [t.name for t in thread.tags]
def test_gmail_label_sync(db, default_account, message, thread, folder, imapuid): if default_account.important_folder is not None: db.session.delete(default_account.important_folder) msg_uid = imapuid.msg_uid # Note that IMAPClient parses numeric labels into integer types. We have to # correctly handle those too. new_flags = {msg_uid: GmailFlags((), (u"\\Important", u"\\Starred", u"foo", 42))} update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid], new_flags) thread_tag_names = {tag.name for tag in thread.tags} assert {"important", "starred", "foo"}.issubset(thread_tag_names)
def refresh_flags_impl(self, crispin_client, max_uids): crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) with session_scope(self.namespace_id) as db_session: local_uids = common.local_uids( account_id=self.account_id, session=db_session, folder_id=self.folder_id, limit=max_uids ) flags = crispin_client.flags(local_uids) expunged_uids = set(local_uids).difference(flags.keys()) common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids) with session_scope(self.namespace_id) as db_session: common.update_metadata(self.account_id, self.folder_id, flags, db_session)
def update_metadata(self, crispin_client, updated): """ Update flags (the only metadata that can change). """ # bigger chunk because the data being fetched here is very small for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE): new_flags = crispin_client.flags(uids) # Messages can disappear in the meantime; we'll update them next # sync. uids = [uid for uid in uids if uid in new_flags] with self.syncmanager_lock: with mailsync_session_scope() as db_session: common.update_metadata(self.account_id, db_session, self.folder_name, self.folder_id, uids, new_flags) db_session.commit()
def test_gmail_label_sync(db, default_account, message, folder, imapuid, default_namespace): # Note that IMAPClient parses numeric labels into integer types. We have to # correctly handle those too. new_flags = { imapuid.msg_uid: GmailFlags((), (u'\\Important', u'\\Starred', u'foo', 42), None) } update_metadata(default_namespace.account.id, folder.id, folder.canonical_name, new_flags, db.session) category_canonical_names = {c.name for c in message.categories} category_display_names = {c.display_name for c in message.categories} assert 'important' in category_canonical_names assert {'foo', '42'}.issubset(category_display_names)
def update_metadata(self, crispin_client, updated): """ Update flags (the only metadata that can change). """ # bigger chunk because the data being fetched here is very small for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE): new_flags = crispin_client.flags(uids) # Messages can disappear in the meantime; we'll update them next # sync. uids = [uid for uid in uids if uid in new_flags] with self.syncmanager_lock: with mailsync_session_scope() as db_session: common.update_metadata(self.account_id, db_session, self.folder_name, self.folder_id, uids, new_flags) db_session.commit()
def condstore_refresh_flags(self, crispin_client): new_highestmodseq = crispin_client.conn.folder_status( self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ'] # Ensure that we have an initial highestmodseq value stored before we # begin polling for changes. if self.highestmodseq is None: self.highestmodseq = new_highestmodseq if new_highestmodseq == self.highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < self.highestmodseq: # This should really never happen, but if it does, handle it. log.warning( 'got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=self.highestmodseq) return # Highestmodseq has changed, update accordingly. crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) changed_flags = crispin_client.condstore_changed_flags( self.highestmodseq) remote_uids = crispin_client.all_uids() with session_scope(self.namespace_id) as db_session: common.update_metadata(self.account_id, self.folder_id, changed_flags, db_session) local_uids = common.local_uids(self.account_id, db_session, self.folder_id) expunged_uids = set(local_uids).difference(remote_uids) if expunged_uids: # If new UIDs have appeared since we last checked in # get_new_uids, save them first. We want to always have the # latest UIDs before expunging anything, in order to properly # capture draft revisions. with session_scope(self.namespace_id) as db_session: lastseenuid = common.lastseenuid(self.account_id, db_session, self.folder_id) if remote_uids and lastseenuid < max(remote_uids): log.info('Downloading new UIDs before expunging') self.get_new_uids(crispin_client) with session_scope(self.namespace_id) as db_session: common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids, db_session) db_session.commit() self.highestmodseq = new_highestmodseq
def refresh_flags_impl(self, crispin_client, max_uids): crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) with session_scope(self.namespace_id) as db_session: local_uids = common.local_uids(account_id=self.account_id, session=db_session, folder_id=self.folder_id, limit=max_uids) flags = crispin_client.flags(local_uids) expunged_uids = set(local_uids).difference(flags.keys()) with session_scope(self.namespace_id) as db_session: common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids, db_session) common.update_metadata(self.account_id, self.folder_id, flags, db_session)
def test_unread_and_draft_tags_applied(db, thread, message, folder, imapuid): """Test that the unread and draft tags are added/removed from a thread after UID flag changes.""" msg_uid = imapuid.msg_uid update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid], {msg_uid: GmailFlags((u'\\Seen', ), (u'\\Draft', ))}) assert 'unread' not in [t.name for t in thread.tags] assert 'drafts' in [t.name for t in thread.tags] assert message.is_read update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid], {msg_uid: GmailFlags((), ())}) assert 'unread' in [t.name for t in thread.tags] assert 'drafts' not in [t.name for t in thread.tags] assert not message.is_read
def test_gmail_label_sync(db, default_account, message, folder, imapuid, default_namespace): msg_uid = imapuid.msg_uid # Note that IMAPClient parses numeric labels into integer types. We have to # correctly handle those too. new_flags = { msg_uid: GmailFlags((), (u'\\Important', u'\\Starred', u'foo', 42)) } update_metadata(default_namespace.account.id, db.session, folder.name, folder.id, [msg_uid], new_flags) category_canonical_names = {c.name for c in message.categories} category_display_names = {c.display_name for c in message.categories} assert 'important' in category_canonical_names assert {'foo', '42'}.issubset(category_display_names)
def test_unread_and_draft_tags_applied(db, thread, message, folder, imapuid): """Test that the unread and draft tags are added/removed from a thread after UID flag changes.""" msg_uid = imapuid.msg_uid update_metadata( ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid], {msg_uid: GmailFlags((u"\\Seen",), (u"\\Draft",))} ) assert "unread" not in [t.name for t in thread.tags] assert "drafts" in [t.name for t in thread.tags] assert message.is_read update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid], {msg_uid: GmailFlags((), ())}) assert "unread" in [t.name for t in thread.tags] assert "drafts" not in [t.name for t in thread.tags] assert not message.is_read
def test_renamed_label_refresh(db, default_account, thread, message, imapuid, folder, mock_imapclient, monkeypatch): # Check that imapuids see their labels refreshed after running # the LabelRenameHandler. msg_uid = imapuid.msg_uid uid_dict = {msg_uid: GmailFlags((), ('stale label', ), ('23', ))} update_metadata(default_account.id, folder.id, folder.canonical_name, uid_dict, db.session) new_flags = { msg_uid: { 'FLAGS': ('\\Seen', ), 'X-GM-LABELS': ('new label', ), 'MODSEQ': ('23', ) } } mock_imapclient._data['[Gmail]/All mail'] = new_flags mock_imapclient.add_folder_data(folder.name, new_flags) monkeypatch.setattr(MockIMAPClient, 'search', lambda x, y: [msg_uid]) semaphore = Semaphore(value=1) rename_handler = LabelRenameHandler(default_account.id, default_account.namespace.id, 'new label', semaphore) # Acquire the semaphore to check that LabelRenameHandlers block if # the semaphore is in-use. semaphore.acquire() rename_handler.start() # Wait 10 secs and check that the data hasn't changed. gevent.sleep(10) labels = list(imapuid.labels) assert len(labels) == 1 assert labels[0].name == 'stale label' semaphore.release() rename_handler.join() db.session.refresh(imapuid) # Now check that the label got updated. labels = list(imapuid.labels) assert len(labels) == 1 assert labels[0].name == 'new label'
def condstore_refresh_flags(self, crispin_client): new_highestmodseq = crispin_client.conn.folder_status( self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ'] # Ensure that we have an initial highestmodseq value stored before we # begin polling for changes. if self.highestmodseq is None: self.highestmodseq = new_highestmodseq if new_highestmodseq == self.highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < self.highestmodseq: # This should really never happen, but if it does, handle it. log.warning('got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=self.highestmodseq) return # Highestmodseq has changed, update accordingly. crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) changed_flags = crispin_client.condstore_changed_flags( self.highestmodseq) remote_uids = crispin_client.all_uids() with session_scope(self.namespace_id) as db_session: common.update_metadata(self.account_id, self.folder_id, changed_flags, db_session) local_uids = common.local_uids(self.account_id, db_session, self.folder_id) expunged_uids = set(local_uids).difference(remote_uids) if expunged_uids: # If new UIDs have appeared since we last checked in # get_new_uids, save them first. We want to always have the # latest UIDs before expunging anything, in order to properly # capture draft revisions. with session_scope(self.namespace_id) as db_session: lastseenuid = common.lastseenuid(self.account_id, db_session, self.folder_id) if remote_uids and lastseenuid < max(remote_uids): log.info('Downloading new UIDs before expunging') self.get_new_uids(crispin_client) with session_scope(self.namespace_id) as db_session: common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids, db_session) db_session.commit() self.highestmodseq = new_highestmodseq
def test_gmail_label_sync(db, default_account, message, thread, folder, imapuid): if default_account.important_folder is not None: db.session.delete(default_account.important_folder) msg_uid = imapuid.msg_uid # Note that IMAPClient parses numeric labels into integer types. We have to # correctly handle those too. new_flags = { msg_uid: GmailFlags((), (u'\\Important', u'\\Starred', u'foo', 42)) } update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid], new_flags) thread_tag_names = {tag.name for tag in thread.tags} assert {'important', 'starred', 'foo'}.issubset(thread_tag_names)
def test_messages_deleted_asynchronously(db, default_account, thread, message, imapuid, folder): msg_uid = imapuid.msg_uid update_metadata( default_account.id, folder.id, folder.canonical_name, {msg_uid: GmailFlags((), ("label", ), None)}, db.session, ) assert "label" in [cat.display_name for cat in message.categories] remove_deleted_uids(default_account.id, folder.id, [msg_uid]) db.session.expire_all() assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2 # Check that message categories do get updated synchronously. assert "label" not in [cat.display_name for cat in message.categories]
def test_renamed_label_refresh(db, default_account, thread, message, imapuid, folder, mock_imapclient, monkeypatch): # Check that imapuids see their labels refreshed after running # the LabelRenameHandler. msg_uid = imapuid.msg_uid uid_dict = {msg_uid: GmailFlags((), ("stale label", ), ("23", ))} update_metadata(default_account.id, folder.id, folder.canonical_name, uid_dict, db.session) new_flags = { msg_uid: { b"FLAGS": (b"\\Seen", ), b"X-GM-LABELS": (b"new label", ), b"MODSEQ": (23, ), } } mock_imapclient._data["[Gmail]/All mail"] = new_flags mock_imapclient.add_folder_data(folder.name, new_flags) monkeypatch.setattr(MockIMAPClient, "search", lambda x, y: [msg_uid]) semaphore = Semaphore(value=1) rename_handler = LabelRenameHandler(default_account.id, default_account.namespace.id, "new label", semaphore) # Acquire the semaphore to check that LabelRenameHandlers block if # the semaphore is in-use. semaphore.acquire() rename_handler.start() gevent.sleep(0) # yield to the handler labels = list(imapuid.labels) assert len(labels) == 1 assert labels[0].name == "stale label" semaphore.release() rename_handler.join() db.session.refresh(imapuid) # Now check that the label got updated. labels = list(imapuid.labels) assert len(labels) == 1 assert labels[0].name == "new label"
def _run_impl(self): self.log.info("Starting LabelRenameHandler", label_name=self.label_name) self.semaphore.acquire(blocking=True) try: with connection_pool(self.account_id).get() as crispin_client: folder_names = [] with session_scope(self.account_id) as db_session: folders = db_session.query(Folder).filter( Folder.account_id == self.account_id) folder_names = [folder.name for folder in folders] db_session.expunge_all() for folder_name in folder_names: crispin_client.select_folder(folder_name, uidvalidity_cb) found_uids = crispin_client.search_uids( ["X-GM-LABELS", utf7_encode(self.label_name)]) for chnk in chunk(found_uids, 200): flags = crispin_client.flags(chnk) self.log.info( "Running metadata update for folder", folder_name=folder_name, ) with session_scope(self.account_id) as db_session: fld = (db_session.query(Folder).options( load_only("id")).filter( Folder.account_id == self.account_id, Folder.name == folder_name, ).one()) common.update_metadata( self.account_id, fld.id, fld.canonical_name, flags, db_session, ) db_session.commit() finally: self.semaphore.release()
def test_deleted_labels_get_gced(empty_db, default_account, thread, message, imapuid, folder): # Check that only the labels without messages attached to them # get deleted. default_namespace = default_account.namespace # Create a label w/ no messages attached. label = Label.find_or_create(empty_db.session, default_account, "dangling label") label.deleted_at = datetime.utcnow() label.category.deleted_at = datetime.utcnow() label_id = label.id empty_db.session.commit() # Create a label with attached messages. msg_uid = imapuid.msg_uid update_metadata( default_account.id, folder.id, folder.canonical_name, {msg_uid: GmailFlags((), ("label", ), None)}, empty_db.session, ) label_ids = [] for cat in message.categories: for l in cat.labels: label_ids.append(l.id) handler = DeleteHandler( account_id=default_account.id, namespace_id=default_namespace.id, provider_name=default_account.provider, uid_accessor=lambda m: m.imapuids, message_ttl=0, ) handler.gc_deleted_categories() empty_db.session.commit() # Check that the first label got gc'ed marked_deleted = empty_db.session.query(Label).get(label_id) assert marked_deleted is None # Check that the other labels didn't. for label_id in label_ids: assert empty_db.session.query(Label).get(label_id) is not None
def test_gmail_label_sync(db): folder = db.session.query(Folder).filter( Folder.account_id == ACCOUNT_ID, Folder.name == '[Gmail]/All Mail').one() account = db.session.query(Account).get(ACCOUNT_ID) if account.important_folder is not None: db.session.delete(account.important_folder) thread = db.session.query(Thread).get(1) message = add_fake_message(db.session, NAMESPACE_ID, thread) db.session.add(ImapUid(account_id=ACCOUNT_ID, message=message, msg_uid=22222, folder=folder)) db.session.commit() new_flags = {22222: GmailFlags((), (u'\\Important', u'\\Starred', u'foo'))} update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [22222], new_flags) thread_tag_names = {tag.name for tag in thread.tags} assert {'important', 'starred', 'foo'}.issubset(thread_tag_names)
def refresh_flags_impl(self, crispin_client, max_uids): crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) # Check for any deleted messages. remote_uids = crispin_client.all_uids() with session_scope(self.namespace_id) as db_session: local_uids = common.local_uids(self.account_id, db_session, self.folder_id) expunged_uids = set(local_uids).difference(remote_uids) if expunged_uids: with self.syncmanager_lock: common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids) # Get recent UIDs to monitor for flag changes. with session_scope(self.namespace_id) as db_session: local_uids = common.local_uids( account_id=self.account_id, session=db_session, folder_id=self.folder_id, limit=max_uids, ) flags = crispin_client.flags(local_uids) if max_uids in self.flags_fetch_results and self.flags_fetch_results[ max_uids] == (local_uids, flags): # If the flags fetch response is exactly the same as the last one # we got, then we don't need to persist any changes. # Stopped logging this to reduce overall logging volume # log.debug('Unchanged flags refresh response, ' # 'not persisting changes', max_uids=max_uids) return log.debug("Changed flags refresh response, persisting changes", max_uids=max_uids) expunged_uids = set(local_uids).difference(flags.keys()) with self.syncmanager_lock: common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids) with self.syncmanager_lock, session_scope( self.namespace_id) as db_session: common.update_metadata(self.account_id, self.folder_id, self.folder_role, flags, db_session) self.flags_fetch_results[max_uids] = (local_uids, flags)
def test_renamed_label_refresh(db, default_account, thread, message, imapuid, folder, mock_imapclient, monkeypatch): # Check that imapuids see their labels refreshed after running # the LabelRenameHandler. msg_uid = imapuid.msg_uid uid_dict = {msg_uid: GmailFlags((), ('stale label',), ('23',))} update_metadata(default_account.id, folder.id, folder.canonical_name, uid_dict, db.session) new_flags = {msg_uid: {'FLAGS': ('\\Seen',), 'X-GM-LABELS': ('new label',), 'MODSEQ': ('23',)}} mock_imapclient._data['[Gmail]/All mail'] = new_flags mock_imapclient.add_folder_data(folder.name, new_flags) monkeypatch.setattr(MockIMAPClient, 'search', lambda x, y: [msg_uid]) semaphore = Semaphore(value=1) rename_handler = LabelRenameHandler(default_account.id, default_account.namespace.id, 'new label', semaphore) # Acquire the semaphore to check that LabelRenameHandlers block if # the semaphore is in-use. semaphore.acquire() rename_handler.start() # Wait 10 secs and check that the data hasn't changed. gevent.sleep(10) labels = list(imapuid.labels) assert len(labels) == 1 assert labels[0].name == 'stale label' semaphore.release() rename_handler.join() db.session.refresh(imapuid) # Now check that the label got updated. labels = list(imapuid.labels) assert len(labels) == 1 assert labels[0].name == 'new label'
def test_update_metadata(db): """Check that threads are updated correctly when a label that we haven't seen before is added to multiple threads -- previously, this would with an IntegrityError because autoflush was disabled.""" first_thread = db.session.query(Thread).get(1) second_thread = db.session.query(Thread).get(2) folder = db.session.query(Folder).filter( Folder.account_id == ACCOUNT_ID, Folder.name == '[Gmail]/All Mail').one() uids = [] first_thread_uids = (22222, 22223) for msg_uid in first_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, first_thread) uids.append( ImapUid(account_id=ACCOUNT_ID, message=message, msg_uid=msg_uid, folder=folder)) second_thread_uids = (22224, 22226) for msg_uid in second_thread_uids: message = add_fake_message(db.session, NAMESPACE_ID, second_thread) uids.append( ImapUid(account_id=ACCOUNT_ID, message=message, msg_uid=msg_uid, folder=folder)) db.session.add_all(uids) db.session.commit() msg_uids = first_thread_uids + second_thread_uids new_flags = { msg_uid: GmailFlags((), (u'\\some_new_label', )) for msg_uid in msg_uids } update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, msg_uids, new_flags) db.session.commit() assert 'some_new_label' in [tag.name for tag in first_thread.tags] assert 'some_new_label' in [tag.name for tag in second_thread.tags]
def test_gmail_label_sync(db): folder = db.session.query(Folder).filter( Folder.account_id == ACCOUNT_ID, Folder.name == '[Gmail]/All Mail').one() account = db.session.query(Account).get(ACCOUNT_ID) if account.important_folder is not None: db.session.delete(account.important_folder) thread = db.session.query(Thread).get(1) message = add_fake_message(db.session, NAMESPACE_ID, thread) db.session.add( ImapUid(account_id=ACCOUNT_ID, message=message, msg_uid=22222, folder=folder)) db.session.commit() new_flags = {22222: GmailFlags((), (u'\\Important', u'\\Starred', u'foo'))} update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [22222], new_flags) thread_tag_names = {tag.name for tag in thread.tags} assert {'important', 'starred', 'foo'}.issubset(thread_tag_names)
def test_deleted_labels_get_gced(db, default_account, thread, message, imapuid, folder): # Check that only the labels without messages attached to them # get deleted. default_namespace = default_account.namespace # Create a label w/ no messages attached. label = Label.find_or_create(db.session, default_account, 'dangling label') label.deleted_at = datetime.utcnow() label.category.deleted_at = datetime.utcnow() label_id = label.id db.session.commit() # Create a label with attached messages. msg_uid = imapuid.msg_uid update_metadata(default_account.id, folder.id, folder.canonical_name, {msg_uid: GmailFlags((), ('label',), None)}, db.session) label_ids = [] for cat in message.categories: for l in cat.labels: label_ids.append(l.id) handler = DeleteHandler(account_id=default_account.id, namespace_id=default_namespace.id, provider_name=default_account.provider, uid_accessor=lambda m: m.imapuids, message_ttl=0) handler.gc_deleted_categories() db.session.commit() # Check that the first label got gc'ed marked_deleted = db.session.query(Label).get(label_id) assert marked_deleted is None # Check that the other labels didn't. for label_id in label_ids: assert db.session.query(Label).get(label_id) is not None
def _run_impl(self): self.log.info('Starting LabelRenameHandler', label_name=self.label_name) self.semaphore.acquire(blocking=True) try: with connection_pool(self.account_id).get() as crispin_client: folder_names = [] with session_scope(self.account_id) as db_session: folders = db_session.query(Folder).filter( Folder.account_id == self.account_id) folder_names = [folder.name for folder in folders] db_session.expunge_all() for folder_name in folder_names: crispin_client.select_folder(folder_name, uidvalidity_cb) found_uids = crispin_client.search_uids(['X-GM-LABELS', utf7_encode(self.label_name)]) for chnk in chunk(found_uids, 200): flags = crispin_client.flags(chnk) self.log.info('Running metadata update for folder', folder_name=folder_name) with session_scope(self.account_id) as db_session: fld = db_session.query(Folder).options(load_only("id"))\ .filter(Folder.account_id == self.account_id, Folder.name == folder_name).one() common.update_metadata(self.account_id, fld.id, fld.canonical_name, flags, db_session) db_session.commit() finally: self.semaphore.release()
def condstore_refresh_flags(self, crispin_client): new_highestmodseq = crispin_client.conn.folder_status( self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ'] # Ensure that we have an initial highestmodseq value stored before we # begin polling for changes. if self.highestmodseq is None: self.highestmodseq = new_highestmodseq if new_highestmodseq == self.highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < self.highestmodseq: # This should really never happen, but if it does, handle it. log.warning('got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=self.highestmodseq) return log.info('HIGHESTMODSEQ has changed, getting changed UIDs', new_highestmodseq=new_highestmodseq, saved_highestmodseq=self.highestmodseq) crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) changed_flags = crispin_client.condstore_changed_flags( self.highestmodseq) remote_uids = crispin_client.all_uids() # In order to be able to sync changes to tens of thousands of flags at # once, we commit updates in batches. We do this in ascending order by # modseq and periodically "checkpoint" our saved highestmodseq. (It's # safe to checkpoint *because* we go in ascending order by modseq.) # That way if the process gets restarted halfway through this refresh, # we don't have to completely start over. It's also slow to load many # objects into the SQLAlchemy session and then issue lots of commits; # we avoid that by batching. flag_batches = chunk( sorted(changed_flags.items(), key=lambda (k, v): v.modseq), CONDSTORE_FLAGS_REFRESH_BATCH_SIZE) for flag_batch in flag_batches: with session_scope(self.namespace_id) as db_session: common.update_metadata(self.account_id, self.folder_id, self.folder_role, dict(flag_batch), db_session) if len(flag_batch) == CONDSTORE_FLAGS_REFRESH_BATCH_SIZE: interim_highestmodseq = max(v.modseq for k, v in flag_batch) self.highestmodseq = interim_highestmodseq with session_scope(self.namespace_id) as db_session: local_uids = common.local_uids(self.account_id, db_session, self.folder_id) expunged_uids = set(local_uids).difference(remote_uids) if expunged_uids: # If new UIDs have appeared since we last checked in # get_new_uids, save them first. We want to always have the # latest UIDs before expunging anything, in order to properly # capture draft revisions. with session_scope(self.namespace_id) as db_session: lastseenuid = common.lastseenuid(self.account_id, db_session, self.folder_id) if remote_uids and lastseenuid < max(remote_uids): log.info('Downloading new UIDs before expunging') self.get_new_uids(crispin_client) common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids) self.highestmodseq = new_highestmodseq