def test_update_metadata(db):
"""Check that threads are updated correctly when a label that we haven't
seen before is added to multiple threads -- previously, this would with an
IntegrityError because autoflush was disabled."""
first_thread = db.session.query(Thread).get(1)
second_thread = db.session.query(Thread).get(2)
folder = db.session.query(Folder).filter(
Folder.account_id == ACCOUNT_ID,
Folder.name == '[Gmail]/All Mail').one()
uids = []
first_thread_uids = (22222, 22223)
for msg_uid in first_thread_uids:
message = add_fake_message(db.session, NAMESPACE_ID, first_thread)
uids.append(ImapUid(account_id=ACCOUNT_ID, message=message,
msg_uid=msg_uid, folder=folder))
second_thread_uids = (22224, 22226)
for msg_uid in second_thread_uids:
message = add_fake_message(db.session, NAMESPACE_ID, second_thread)
uids.append(ImapUid(account_id=ACCOUNT_ID, message=message,
msg_uid=msg_uid, folder=folder))
db.session.add_all(uids)
db.session.commit()
msg_uids = first_thread_uids + second_thread_uids
new_flags = {msg_uid: GmailFlags((), (u'\\some_new_label',))
for msg_uid in msg_uids}
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, msg_uids,
new_flags)
db.session.commit()
assert 'some_new_label' in [tag.name for tag in first_thread.tags]
assert 'some_new_label' in [tag.name for tag in second_thread.tags]
def _run_impl(self):
self.log.info('Starting LabelRenameHandler',
label_name=self.label_name)
with connection_pool(self.account_id).get() as crispin_client:
folder_names = []
with session_scope(self.account_id) as db_session:
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id)
folder_names = [folder.name for folder in folders]
db_session.expunge_all()
for folder_name in folder_names:
crispin_client.select_folder(folder_name, uidvalidity_cb)
found_uids = crispin_client.search_uids(
['X-GM-LABELS', utf7_encode(self.label_name)])
flags = crispin_client.flags(found_uids)
self.log.info('Running metadata update for folder',
folder_name=folder_name)
with session_scope(self.account_id) as db_session:
common.update_metadata(self.account_id, folder.id, flags,
db_session)
db_session.commit()
def refresh_flags_impl(self, crispin_client, max_uids):
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(account_id=self.account_id,
session=db_session,
folder_id=self.folder_id,
limit=max_uids)
flags = crispin_client.flags(local_uids)
if (max_uids in self.flags_fetch_results and
self.flags_fetch_results[max_uids] == (local_uids, flags)):
# If the flags fetch response is exactly the same as the last one
# we got, then we don't need to persist any changes.
log.debug('Unchanged flags refresh response, '
'not persisting changes', max_uids=max_uids)
return
log.debug('Changed flags refresh response, persisting changes',
max_uids=max_uids)
expunged_uids = set(local_uids).difference(flags.keys())
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids)
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
self.folder_role, flags, db_session)
self.flags_fetch_results[max_uids] = (local_uids, flags)
def test_renamed_label_refresh(db, default_account, thread, message,
imapuid, folder, mock_imapclient, monkeypatch):
# Check that imapuids see their labels refreshed after running
# the LabelRenameHandler.
msg_uid = imapuid.msg_uid
uid_dict = {msg_uid: GmailFlags((), ('stale label',))}
update_metadata(default_account.id, folder.id, uid_dict, db.session)
new_flags = {msg_uid: {'FLAGS': ('\\Seen',), 'X-GM-LABELS': ('new label',)}}
mock_imapclient._data['[Gmail]/All mail'] = new_flags
mock_imapclient.add_folder_data(folder.name, new_flags)
monkeypatch.setattr(MockIMAPClient, 'search',
lambda x, y: [msg_uid])
rename_handler = LabelRenameHandler(default_account.id,
default_account.namespace.id,
'new label')
rename_handler.start()
rename_handler.join()
labels = list(imapuid.labels)
assert len(labels) == 1
assert labels[0].name == 'new label'
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
# Highestmodseq has changed, update accordingly.
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
with session_scope() as db_session:
common.update_metadata(self.account_id, self.folder_id,
changed_flags, db_session)
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
db_session.commit()
self.highestmodseq = new_highestmodseq
def test_update_metadata(db, folder):
"""Check that threads are updated correctly when a label that we haven't
seen before is added to multiple threads -- previously, this would fail
with an IntegrityError because autoflush was disabled."""
first_thread = add_fake_thread(db.session, NAMESPACE_ID)
second_thread = add_fake_thread(db.session, NAMESPACE_ID)
uids = []
first_thread_uids = (22222, 22223)
for msg_uid in first_thread_uids:
message = add_fake_message(db.session, NAMESPACE_ID, first_thread)
uids.append(add_fake_imapuid(db.session, ACCOUNT_ID, message, folder, msg_uid))
second_thread_uids = (22224, 22226)
for msg_uid in second_thread_uids:
message = add_fake_message(db.session, NAMESPACE_ID, second_thread)
uids.append(add_fake_imapuid(db.session, ACCOUNT_ID, message, folder, msg_uid))
db.session.add_all(uids)
db.session.commit()
msg_uids = first_thread_uids + second_thread_uids
new_flags = {msg_uid: GmailFlags((), (u"\\some_new_label",)) for msg_uid in msg_uids}
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, msg_uids, new_flags)
db.session.commit()
assert "some_new_label" in [tag.name for tag in first_thread.tags]
assert "some_new_label" in [tag.name for tag in second_thread.tags]
def test_update_metadata(db, folder):
"""Check that threads are updated correctly when a label that we haven't
seen before is added to multiple threads -- previously, this would fail
with an IntegrityError because autoflush was disabled."""
first_thread = add_fake_thread(db.session, NAMESPACE_ID)
second_thread = add_fake_thread(db.session, NAMESPACE_ID)
uids = []
first_thread_uids = (22222, 22223)
for msg_uid in first_thread_uids:
message = add_fake_message(db.session, NAMESPACE_ID, first_thread)
uids.append(
add_fake_imapuid(db.session, ACCOUNT_ID, message, folder, msg_uid))
second_thread_uids = (22224, 22226)
for msg_uid in second_thread_uids:
message = add_fake_message(db.session, NAMESPACE_ID, second_thread)
uids.append(
add_fake_imapuid(db.session, ACCOUNT_ID, message, folder, msg_uid))
db.session.add_all(uids)
db.session.commit()
msg_uids = first_thread_uids + second_thread_uids
new_flags = {
msg_uid: GmailFlags((), (u'\\some_new_label', ))
for msg_uid in msg_uids
}
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, msg_uids,
new_flags)
db.session.commit()
assert 'some_new_label' in [tag.name for tag in first_thread.tags]
assert 'some_new_label' in [tag.name for tag in second_thread.tags]
def test_gmail_drafts_flag_constrained_by_folder(
db, default_account, message, imapuid, folder
):
new_flags = {imapuid.msg_uid: GmailFlags((), (u"\\Draft",), None)}
update_metadata(default_account.id, folder.id, "all", new_flags, db.session)
assert message.is_draft
update_metadata(default_account.id, folder.id, "trash", new_flags, db.session)
assert not message.is_draft
def test_gmail_drafts_flag_constrained_by_folder(db, default_account, message,
imapuid, folder):
new_flags = {imapuid.msg_uid: GmailFlags((), (u'\\Draft',), None)}
update_metadata(default_account.id, folder.id, 'all', new_flags,
db.session)
assert message.is_draft
update_metadata(default_account.id, folder.id, 'trash', new_flags,
db.session)
assert not message.is_draft
def test_gmail_label_sync(db, default_account, message, folder, imapuid, default_namespace):
# Note that IMAPClient parses numeric labels into integer types. We have to
# correctly handle those too.
new_flags = {imapuid.msg_uid: GmailFlags((), (u"\\Important", u"\\Starred", u"foo", 42), None)}
update_metadata(default_namespace.account.id, folder.id, folder.canonical_name, new_flags, db.session)
category_canonical_names = {c.name for c in message.categories}
category_display_names = {c.display_name for c in message.categories}
assert "important" in category_canonical_names
assert {"foo", "42"}.issubset(category_display_names)
def test_messages_deleted_asynchronously(db, default_account, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(default_account.id, db.session, folder.name, folder.id,
[msg_uid], {msg_uid: GmailFlags((), ('label',))})
assert 'label' in [cat.display_name for cat in message.categories]
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2
# Check that message categories do get updated synchronously.
assert 'label' not in [cat.display_name for cat in message.categories]
def test_generic_drafts_flag_constrained_by_folder(db, generic_account, folder_role):
msg_uid = 22
thread = add_fake_thread(db.session, generic_account.namespace.id)
message = add_fake_message(db.session, generic_account.namespace.id, thread)
folder = add_fake_folder(db.session, generic_account)
add_fake_imapuid(db.session, generic_account.id, message, folder, msg_uid)
new_flags = {msg_uid: Flags(("\\Draft",), None)}
update_metadata(generic_account.id, folder.id, folder_role, new_flags, db.session)
assert message.is_draft == (folder_role == "drafts")
def test_messages_deleted_asynchronously(db, default_account, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(default_account.id, folder.id,
{msg_uid: GmailFlags((), ('label', ))}, db.session)
assert 'label' in [cat.display_name for cat in message.categories]
remove_deleted_uids(default_account.id, folder.id, [msg_uid], db.session)
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2
# Check that message categories do get updated synchronously.
assert 'label' not in [cat.display_name for cat in message.categories]
def test_generic_drafts_flag_constrained_by_folder(db, generic_account, folder_role):
msg_uid = 22
thread = add_fake_thread(db.session, generic_account.namespace.id)
message = add_fake_message(db.session, generic_account.namespace.id, thread)
folder = add_fake_folder(db.session, generic_account)
add_fake_imapuid(db.session, generic_account.id, message, folder, msg_uid)
new_flags = {msg_uid: Flags((b"\\Draft",), None)}
update_metadata(generic_account.id, folder.id, folder_role, new_flags, db.session)
assert message.is_draft == (folder_role == "drafts")
def test_only_uids_deleted_synchronously(db, default_account,
default_namespace, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(default_account.id, db.session, folder.name, folder.id,
[msg_uid], {msg_uid: GmailFlags((), ('label', ))})
assert 'label' in [t.name for t in thread.tags]
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 1
# Check that thread tags do get updated synchronously.
assert 'label' not in [t.name for t in thread.tags]
def test_only_uids_deleted_synchronously(db, default_account,
default_namespace, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(default_account.id, db.session, folder.name, folder.id,
[msg_uid], {msg_uid: GmailFlags((), ('label',))})
assert 'label' in [t.name for t in thread.tags]
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2
# Check that thread tags do get updated synchronously.
assert 'label' not in [t.name for t in thread.tags]
def test_gmail_label_sync(db, default_account, message, thread, folder, imapuid):
if default_account.important_folder is not None:
db.session.delete(default_account.important_folder)
msg_uid = imapuid.msg_uid
# Note that IMAPClient parses numeric labels into integer types. We have to
# correctly handle those too.
new_flags = {msg_uid: GmailFlags((), (u"\\Important", u"\\Starred", u"foo", 42))}
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid], new_flags)
thread_tag_names = {tag.name for tag in thread.tags}
assert {"important", "starred", "foo"}.issubset(thread_tag_names)
def refresh_flags_impl(self, crispin_client, max_uids):
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(
account_id=self.account_id, session=db_session, folder_id=self.folder_id, limit=max_uids
)
flags = crispin_client.flags(local_uids)
expunged_uids = set(local_uids).difference(flags.keys())
common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids)
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id, flags, db_session)
def update_metadata(self, crispin_client, updated):
""" Update flags (the only metadata that can change). """
# bigger chunk because the data being fetched here is very small
for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE):
new_flags = crispin_client.flags(uids)
# Messages can disappear in the meantime; we'll update them next
# sync.
uids = [uid for uid in uids if uid in new_flags]
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
common.update_metadata(self.account_id, db_session,
self.folder_name, self.folder_id,
uids, new_flags)
db_session.commit()
def test_gmail_label_sync(db, default_account, message, folder, imapuid,
default_namespace):
# Note that IMAPClient parses numeric labels into integer types. We have to
# correctly handle those too.
new_flags = {
imapuid.msg_uid:
GmailFlags((), (u'\\Important', u'\\Starred', u'foo', 42), None)
}
update_metadata(default_namespace.account.id, folder.id,
folder.canonical_name, new_flags, db.session)
category_canonical_names = {c.name for c in message.categories}
category_display_names = {c.display_name for c in message.categories}
assert 'important' in category_canonical_names
assert {'foo', '42'}.issubset(category_display_names)
def update_metadata(self, crispin_client, updated):
""" Update flags (the only metadata that can change). """
# bigger chunk because the data being fetched here is very small
for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE):
new_flags = crispin_client.flags(uids)
# Messages can disappear in the meantime; we'll update them next
# sync.
uids = [uid for uid in uids if uid in new_flags]
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
common.update_metadata(self.account_id, db_session,
self.folder_name, self.folder_id,
uids, new_flags)
db_session.commit()
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning(
'got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
# Highestmodseq has changed, update accordingly.
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
changed_flags, db_session)
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
# If new UIDs have appeared since we last checked in
# get_new_uids, save them first. We want to always have the
# latest UIDs before expunging anything, in order to properly
# capture draft revisions.
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
if remote_uids and lastseenuid < max(remote_uids):
log.info('Downloading new UIDs before expunging')
self.get_new_uids(crispin_client)
with session_scope(self.namespace_id) as db_session:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
db_session.commit()
self.highestmodseq = new_highestmodseq
def refresh_flags_impl(self, crispin_client, max_uids):
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(account_id=self.account_id,
session=db_session,
folder_id=self.folder_id,
limit=max_uids)
flags = crispin_client.flags(local_uids)
expunged_uids = set(local_uids).difference(flags.keys())
with session_scope(self.namespace_id) as db_session:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
common.update_metadata(self.account_id, self.folder_id, flags,
db_session)
def test_unread_and_draft_tags_applied(db, thread, message, folder, imapuid):
"""Test that the unread and draft tags are added/removed from a thread
after UID flag changes."""
msg_uid = imapuid.msg_uid
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid],
{msg_uid: GmailFlags((u'\\Seen', ), (u'\\Draft', ))})
assert 'unread' not in [t.name for t in thread.tags]
assert 'drafts' in [t.name for t in thread.tags]
assert message.is_read
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid],
{msg_uid: GmailFlags((), ())})
assert 'unread' in [t.name for t in thread.tags]
assert 'drafts' not in [t.name for t in thread.tags]
assert not message.is_read
def test_gmail_label_sync(db, default_account, message, folder,
imapuid, default_namespace):
msg_uid = imapuid.msg_uid
# Note that IMAPClient parses numeric labels into integer types. We have to
# correctly handle those too.
new_flags = {
msg_uid: GmailFlags((), (u'\\Important', u'\\Starred', u'foo', 42))
}
update_metadata(default_namespace.account.id, db.session, folder.name,
folder.id, [msg_uid], new_flags)
category_canonical_names = {c.name for c in message.categories}
category_display_names = {c.display_name for c in message.categories}
assert 'important' in category_canonical_names
assert {'foo', '42'}.issubset(category_display_names)
def test_unread_and_draft_tags_applied(db, thread, message, folder, imapuid):
"""Test that the unread and draft tags are added/removed from a thread
after UID flag changes."""
msg_uid = imapuid.msg_uid
update_metadata(
ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid], {msg_uid: GmailFlags((u"\\Seen",), (u"\\Draft",))}
)
assert "unread" not in [t.name for t in thread.tags]
assert "drafts" in [t.name for t in thread.tags]
assert message.is_read
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid], {msg_uid: GmailFlags((), ())})
assert "unread" in [t.name for t in thread.tags]
assert "drafts" not in [t.name for t in thread.tags]
assert not message.is_read
def test_renamed_label_refresh(db, default_account, thread, message, imapuid,
folder, mock_imapclient, monkeypatch):
# Check that imapuids see their labels refreshed after running
# the LabelRenameHandler.
msg_uid = imapuid.msg_uid
uid_dict = {msg_uid: GmailFlags((), ('stale label', ), ('23', ))}
update_metadata(default_account.id, folder.id, folder.canonical_name,
uid_dict, db.session)
new_flags = {
msg_uid: {
'FLAGS': ('\\Seen', ),
'X-GM-LABELS': ('new label', ),
'MODSEQ': ('23', )
}
}
mock_imapclient._data['[Gmail]/All mail'] = new_flags
mock_imapclient.add_folder_data(folder.name, new_flags)
monkeypatch.setattr(MockIMAPClient, 'search', lambda x, y: [msg_uid])
semaphore = Semaphore(value=1)
rename_handler = LabelRenameHandler(default_account.id,
default_account.namespace.id,
'new label', semaphore)
# Acquire the semaphore to check that LabelRenameHandlers block if
# the semaphore is in-use.
semaphore.acquire()
rename_handler.start()
# Wait 10 secs and check that the data hasn't changed.
gevent.sleep(10)
labels = list(imapuid.labels)
assert len(labels) == 1
assert labels[0].name == 'stale label'
semaphore.release()
rename_handler.join()
db.session.refresh(imapuid)
# Now check that the label got updated.
labels = list(imapuid.labels)
assert len(labels) == 1
assert labels[0].name == 'new label'
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
# Highestmodseq has changed, update accordingly.
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
changed_flags, db_session)
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
# If new UIDs have appeared since we last checked in
# get_new_uids, save them first. We want to always have the
# latest UIDs before expunging anything, in order to properly
# capture draft revisions.
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
if remote_uids and lastseenuid < max(remote_uids):
log.info('Downloading new UIDs before expunging')
self.get_new_uids(crispin_client)
with session_scope(self.namespace_id) as db_session:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
db_session.commit()
self.highestmodseq = new_highestmodseq
def test_gmail_label_sync(db, default_account, message, thread, folder,
imapuid):
if default_account.important_folder is not None:
db.session.delete(default_account.important_folder)
msg_uid = imapuid.msg_uid
# Note that IMAPClient parses numeric labels into integer types. We have to
# correctly handle those too.
new_flags = {
msg_uid: GmailFlags((), (u'\\Important', u'\\Starred', u'foo', 42))
}
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [msg_uid],
new_flags)
thread_tag_names = {tag.name for tag in thread.tags}
assert {'important', 'starred', 'foo'}.issubset(thread_tag_names)
def test_messages_deleted_asynchronously(db, default_account, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(
default_account.id,
folder.id,
folder.canonical_name,
{msg_uid: GmailFlags((), ("label", ), None)},
db.session,
)
assert "label" in [cat.display_name for cat in message.categories]
remove_deleted_uids(default_account.id, folder.id, [msg_uid])
db.session.expire_all()
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2
# Check that message categories do get updated synchronously.
assert "label" not in [cat.display_name for cat in message.categories]
def test_renamed_label_refresh(db, default_account, thread, message, imapuid,
folder, mock_imapclient, monkeypatch):
# Check that imapuids see their labels refreshed after running
# the LabelRenameHandler.
msg_uid = imapuid.msg_uid
uid_dict = {msg_uid: GmailFlags((), ("stale label", ), ("23", ))}
update_metadata(default_account.id, folder.id, folder.canonical_name,
uid_dict, db.session)
new_flags = {
msg_uid: {
b"FLAGS": (b"\\Seen", ),
b"X-GM-LABELS": (b"new label", ),
b"MODSEQ": (23, ),
}
}
mock_imapclient._data["[Gmail]/All mail"] = new_flags
mock_imapclient.add_folder_data(folder.name, new_flags)
monkeypatch.setattr(MockIMAPClient, "search", lambda x, y: [msg_uid])
semaphore = Semaphore(value=1)
rename_handler = LabelRenameHandler(default_account.id,
default_account.namespace.id,
"new label", semaphore)
# Acquire the semaphore to check that LabelRenameHandlers block if
# the semaphore is in-use.
semaphore.acquire()
rename_handler.start()
gevent.sleep(0) # yield to the handler
labels = list(imapuid.labels)
assert len(labels) == 1
assert labels[0].name == "stale label"
semaphore.release()
rename_handler.join()
db.session.refresh(imapuid)
# Now check that the label got updated.
labels = list(imapuid.labels)
assert len(labels) == 1
assert labels[0].name == "new label"
def _run_impl(self):
self.log.info("Starting LabelRenameHandler",
label_name=self.label_name)
self.semaphore.acquire(blocking=True)
try:
with connection_pool(self.account_id).get() as crispin_client:
folder_names = []
with session_scope(self.account_id) as db_session:
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id)
folder_names = [folder.name for folder in folders]
db_session.expunge_all()
for folder_name in folder_names:
crispin_client.select_folder(folder_name, uidvalidity_cb)
found_uids = crispin_client.search_uids(
["X-GM-LABELS",
utf7_encode(self.label_name)])
for chnk in chunk(found_uids, 200):
flags = crispin_client.flags(chnk)
self.log.info(
"Running metadata update for folder",
folder_name=folder_name,
)
with session_scope(self.account_id) as db_session:
fld = (db_session.query(Folder).options(
load_only("id")).filter(
Folder.account_id == self.account_id,
Folder.name == folder_name,
).one())
common.update_metadata(
self.account_id,
fld.id,
fld.canonical_name,
flags,
db_session,
)
db_session.commit()
finally:
self.semaphore.release()
def test_deleted_labels_get_gced(empty_db, default_account, thread, message,
imapuid, folder):
# Check that only the labels without messages attached to them
# get deleted.
default_namespace = default_account.namespace
# Create a label w/ no messages attached.
label = Label.find_or_create(empty_db.session, default_account,
"dangling label")
label.deleted_at = datetime.utcnow()
label.category.deleted_at = datetime.utcnow()
label_id = label.id
empty_db.session.commit()
# Create a label with attached messages.
msg_uid = imapuid.msg_uid
update_metadata(
default_account.id,
folder.id,
folder.canonical_name,
{msg_uid: GmailFlags((), ("label", ), None)},
empty_db.session,
)
label_ids = []
for cat in message.categories:
for l in cat.labels:
label_ids.append(l.id)
handler = DeleteHandler(
account_id=default_account.id,
namespace_id=default_namespace.id,
provider_name=default_account.provider,
uid_accessor=lambda m: m.imapuids,
message_ttl=0,
)
handler.gc_deleted_categories()
empty_db.session.commit()
# Check that the first label got gc'ed
marked_deleted = empty_db.session.query(Label).get(label_id)
assert marked_deleted is None
# Check that the other labels didn't.
for label_id in label_ids:
assert empty_db.session.query(Label).get(label_id) is not None
def test_gmail_label_sync(db):
folder = db.session.query(Folder).filter(
Folder.account_id == ACCOUNT_ID,
Folder.name == '[Gmail]/All Mail').one()
account = db.session.query(Account).get(ACCOUNT_ID)
if account.important_folder is not None:
db.session.delete(account.important_folder)
thread = db.session.query(Thread).get(1)
message = add_fake_message(db.session, NAMESPACE_ID, thread)
db.session.add(ImapUid(account_id=ACCOUNT_ID, message=message,
msg_uid=22222, folder=folder))
db.session.commit()
new_flags = {22222: GmailFlags((), (u'\\Important', u'\\Starred', u'foo'))}
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [22222],
new_flags)
thread_tag_names = {tag.name for tag in thread.tags}
assert {'important', 'starred', 'foo'}.issubset(thread_tag_names)
def refresh_flags_impl(self, crispin_client, max_uids):
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
# Check for any deleted messages.
remote_uids = crispin_client.all_uids()
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
with self.syncmanager_lock:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids)
# Get recent UIDs to monitor for flag changes.
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(
account_id=self.account_id,
session=db_session,
folder_id=self.folder_id,
limit=max_uids,
)
flags = crispin_client.flags(local_uids)
if max_uids in self.flags_fetch_results and self.flags_fetch_results[
max_uids] == (local_uids, flags):
# If the flags fetch response is exactly the same as the last one
# we got, then we don't need to persist any changes.
# Stopped logging this to reduce overall logging volume
# log.debug('Unchanged flags refresh response, '
# 'not persisting changes', max_uids=max_uids)
return
log.debug("Changed flags refresh response, persisting changes",
max_uids=max_uids)
expunged_uids = set(local_uids).difference(flags.keys())
with self.syncmanager_lock:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids)
with self.syncmanager_lock, session_scope(
self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
self.folder_role, flags, db_session)
self.flags_fetch_results[max_uids] = (local_uids, flags)
def test_renamed_label_refresh(db, default_account, thread, message,
imapuid, folder, mock_imapclient, monkeypatch):
# Check that imapuids see their labels refreshed after running
# the LabelRenameHandler.
msg_uid = imapuid.msg_uid
uid_dict = {msg_uid: GmailFlags((), ('stale label',), ('23',))}
update_metadata(default_account.id, folder.id, folder.canonical_name,
uid_dict, db.session)
new_flags = {msg_uid: {'FLAGS': ('\\Seen',), 'X-GM-LABELS': ('new label',),
'MODSEQ': ('23',)}}
mock_imapclient._data['[Gmail]/All mail'] = new_flags
mock_imapclient.add_folder_data(folder.name, new_flags)
monkeypatch.setattr(MockIMAPClient, 'search',
lambda x, y: [msg_uid])
semaphore = Semaphore(value=1)
rename_handler = LabelRenameHandler(default_account.id,
default_account.namespace.id,
'new label', semaphore)
# Acquire the semaphore to check that LabelRenameHandlers block if
# the semaphore is in-use.
semaphore.acquire()
rename_handler.start()
# Wait 10 secs and check that the data hasn't changed.
gevent.sleep(10)
labels = list(imapuid.labels)
assert len(labels) == 1
assert labels[0].name == 'stale label'
semaphore.release()
rename_handler.join()
db.session.refresh(imapuid)
# Now check that the label got updated.
labels = list(imapuid.labels)
assert len(labels) == 1
assert labels[0].name == 'new label'
def test_update_metadata(db):
"""Check that threads are updated correctly when a label that we haven't
seen before is added to multiple threads -- previously, this would with an
IntegrityError because autoflush was disabled."""
first_thread = db.session.query(Thread).get(1)
second_thread = db.session.query(Thread).get(2)
folder = db.session.query(Folder).filter(
Folder.account_id == ACCOUNT_ID,
Folder.name == '[Gmail]/All Mail').one()
uids = []
first_thread_uids = (22222, 22223)
for msg_uid in first_thread_uids:
message = add_fake_message(db.session, NAMESPACE_ID, first_thread)
uids.append(
ImapUid(account_id=ACCOUNT_ID,
message=message,
msg_uid=msg_uid,
folder=folder))
second_thread_uids = (22224, 22226)
for msg_uid in second_thread_uids:
message = add_fake_message(db.session, NAMESPACE_ID, second_thread)
uids.append(
ImapUid(account_id=ACCOUNT_ID,
message=message,
msg_uid=msg_uid,
folder=folder))
db.session.add_all(uids)
db.session.commit()
msg_uids = first_thread_uids + second_thread_uids
new_flags = {
msg_uid: GmailFlags((), (u'\\some_new_label', ))
for msg_uid in msg_uids
}
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, msg_uids,
new_flags)
db.session.commit()
assert 'some_new_label' in [tag.name for tag in first_thread.tags]
assert 'some_new_label' in [tag.name for tag in second_thread.tags]
def test_gmail_label_sync(db):
folder = db.session.query(Folder).filter(
Folder.account_id == ACCOUNT_ID,
Folder.name == '[Gmail]/All Mail').one()
account = db.session.query(Account).get(ACCOUNT_ID)
if account.important_folder is not None:
db.session.delete(account.important_folder)
thread = db.session.query(Thread).get(1)
message = add_fake_message(db.session, NAMESPACE_ID, thread)
db.session.add(
ImapUid(account_id=ACCOUNT_ID,
message=message,
msg_uid=22222,
folder=folder))
db.session.commit()
new_flags = {22222: GmailFlags((), (u'\\Important', u'\\Starred', u'foo'))}
update_metadata(ACCOUNT_ID, db.session, folder.name, folder.id, [22222],
new_flags)
thread_tag_names = {tag.name for tag in thread.tags}
assert {'important', 'starred', 'foo'}.issubset(thread_tag_names)
def test_deleted_labels_get_gced(db, default_account, thread, message,
imapuid, folder):
# Check that only the labels without messages attached to them
# get deleted.
default_namespace = default_account.namespace
# Create a label w/ no messages attached.
label = Label.find_or_create(db.session, default_account, 'dangling label')
label.deleted_at = datetime.utcnow()
label.category.deleted_at = datetime.utcnow()
label_id = label.id
db.session.commit()
# Create a label with attached messages.
msg_uid = imapuid.msg_uid
update_metadata(default_account.id, folder.id, folder.canonical_name,
{msg_uid: GmailFlags((), ('label',), None)}, db.session)
label_ids = []
for cat in message.categories:
for l in cat.labels:
label_ids.append(l.id)
handler = DeleteHandler(account_id=default_account.id,
namespace_id=default_namespace.id,
provider_name=default_account.provider,
uid_accessor=lambda m: m.imapuids,
message_ttl=0)
handler.gc_deleted_categories()
db.session.commit()
# Check that the first label got gc'ed
marked_deleted = db.session.query(Label).get(label_id)
assert marked_deleted is None
# Check that the other labels didn't.
for label_id in label_ids:
assert db.session.query(Label).get(label_id) is not None
def _run_impl(self):
self.log.info('Starting LabelRenameHandler',
label_name=self.label_name)
self.semaphore.acquire(blocking=True)
try:
with connection_pool(self.account_id).get() as crispin_client:
folder_names = []
with session_scope(self.account_id) as db_session:
folders = db_session.query(Folder).filter(
Folder.account_id == self.account_id)
folder_names = [folder.name for folder in folders]
db_session.expunge_all()
for folder_name in folder_names:
crispin_client.select_folder(folder_name, uidvalidity_cb)
found_uids = crispin_client.search_uids(['X-GM-LABELS',
utf7_encode(self.label_name)])
for chnk in chunk(found_uids, 200):
flags = crispin_client.flags(chnk)
self.log.info('Running metadata update for folder',
folder_name=folder_name)
with session_scope(self.account_id) as db_session:
fld = db_session.query(Folder).options(load_only("id"))\
.filter(Folder.account_id == self.account_id,
Folder.name == folder_name).one()
common.update_metadata(self.account_id, fld.id,
fld.canonical_name, flags,
db_session)
db_session.commit()
finally:
self.semaphore.release()
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
log.info('HIGHESTMODSEQ has changed, getting changed UIDs',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
# In order to be able to sync changes to tens of thousands of flags at
# once, we commit updates in batches. We do this in ascending order by
# modseq and periodically "checkpoint" our saved highestmodseq. (It's
# safe to checkpoint *because* we go in ascending order by modseq.)
# That way if the process gets restarted halfway through this refresh,
# we don't have to completely start over. It's also slow to load many
# objects into the SQLAlchemy session and then issue lots of commits;
# we avoid that by batching.
flag_batches = chunk(
sorted(changed_flags.items(), key=lambda (k, v): v.modseq),
CONDSTORE_FLAGS_REFRESH_BATCH_SIZE)
for flag_batch in flag_batches:
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
self.folder_role, dict(flag_batch),
db_session)
if len(flag_batch) == CONDSTORE_FLAGS_REFRESH_BATCH_SIZE:
interim_highestmodseq = max(v.modseq for k, v in flag_batch)
self.highestmodseq = interim_highestmodseq
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
# If new UIDs have appeared since we last checked in
# get_new_uids, save them first. We want to always have the
# latest UIDs before expunging anything, in order to properly
# capture draft revisions.
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
if remote_uids and lastseenuid < max(remote_uids):
log.info('Downloading new UIDs before expunging')
self.get_new_uids(crispin_client)
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids)
self.highestmodseq = new_highestmodseq
