def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
# Highestmodseq has changed, update accordingly.
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
with session_scope() as db_session:
common.update_metadata(self.account_id, self.folder_id,
changed_flags, db_session)
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
db_session.commit()
self.highestmodseq = new_highestmodseq
def resync_uids_impl(self):
# First, let's check if the UIVDALIDITY change was spurious, if
# it is, just discard it and go on.
with self.conn_pool.get() as crispin_client:
crispin_client.select_folder(self.folder_name, lambda *args: True)
remote_uidvalidity = crispin_client.selected_uidvalidity
remote_uidnext = crispin_client.selected_uidnext
if remote_uidvalidity <= self.uidvalidity:
log.debug('UIDVALIDITY unchanged')
return
# Otherwise, if the UIDVALIDITY really has changed, discard all saved
# UIDs for the folder, mark associated messages for garbage-collection,
# and return to the 'initial' state to resync.
# This will cause message and threads to be deleted and recreated, but
# uidinvalidity is sufficiently rare that this tradeoff is acceptable.
with session_scope(self.namespace_id) as db_session:
invalid_uids = {
uid
for uid, in db_session.query(ImapUid.msg_uid).filter_by(
account_id=self.account_id, folder_id=self.folder_id)
}
common.remove_deleted_uids(self.account_id, self.folder_id,
invalid_uids, db_session)
self.uidvalidity = remote_uidvalidity
self.highestmodseq = None
self.uidnext = remote_uidnext
def refresh_flags_impl(self, crispin_client, max_uids):
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(account_id=self.account_id,
session=db_session,
folder_id=self.folder_id,
limit=max_uids)
flags = crispin_client.flags(local_uids)
if (max_uids in self.flags_fetch_results and
self.flags_fetch_results[max_uids] == (local_uids, flags)):
# If the flags fetch response is exactly the same as the last one
# we got, then we don't need to persist any changes.
log.debug('Unchanged flags refresh response, '
'not persisting changes', max_uids=max_uids)
return
log.debug('Changed flags refresh response, persisting changes',
max_uids=max_uids)
expunged_uids = set(local_uids).difference(flags.keys())
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids)
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
self.folder_role, flags, db_session)
self.flags_fetch_results[max_uids] = (local_uids, flags)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
remote_uids = sorted(crispin_client.all_uids(), key=int)
with self.syncmanager_lock:
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(
self.account_id, db_session, self.folder_id
)
common.remove_deleted_uids(
self.account_id, self.folder_id, set(local_uids) - set(remote_uids)
)
unknown_uids = set(remote_uids) - local_uids
with session_scope(self.namespace_id) as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
download_uid_count=len(unknown_uids),
)
change_poller = gevent.spawn(self.poll_for_changes)
bind_context(change_poller, "changepoller", self.account_id, self.folder_id)
if self.is_all_mail(crispin_client):
# Prioritize UIDs for messages in the inbox folder.
if len(remote_uids) < 1e6:
inbox_uids = set(
crispin_client.search_uids(["X-GM-LABELS", "inbox"])
)
else:
# The search above is really slow (times out) on really
# large mailboxes, so bound the search to messages within
# the past month in order to get anywhere.
since = datetime.utcnow() - timedelta(days=30)
inbox_uids = set(
crispin_client.search_uids(
["X-GM-LABELS", "inbox", "SINCE", since]
)
)
uids_to_download = sorted(unknown_uids - inbox_uids) + sorted(
unknown_uids & inbox_uids
)
else:
uids_to_download = sorted(unknown_uids)
for uids in chunk(reversed(uids_to_download), 1024):
g_metadata = crispin_client.g_metadata(uids)
# UIDs might have been expunged since sync started, in which
# case the g_metadata call above will return nothing.
# They may also have been preemptively downloaded by thread
# expansion. We can omit such UIDs.
uids = [u for u in uids if u in g_metadata and u not in self.saved_uids]
self.batch_download_uids(crispin_client, uids, g_metadata)
finally:
if change_poller is not None:
# schedule change_poller to die
gevent.kill(change_poller)
def test_deleting_from_a_message_with_multiple_uids(db, default_account,
message, thread):
"""Check that deleting a imapuid from a message with
multiple uids doesn't mark the message for deletion."""
inbox_folder = Folder.find_or_create(db.session, default_account, "inbox",
"inbox")
sent_folder = Folder.find_or_create(db.session, default_account, "sent",
"sent")
add_fake_imapuid(db.session, default_account.id, message, sent_folder,
1337)
add_fake_imapuid(db.session, default_account.id, message, inbox_folder,
2222)
assert len(message.imapuids) == 2
remove_deleted_uids(default_account.id, inbox_folder.id, [2222])
db.session.expire_all()
assert (
message.deleted_at is None
), "The associated message should not have been marked for deletion."
assert len(
message.imapuids) == 1, "The message should have only one imapuid."
def resync_uids_impl(self):
# First, let's check if the UIVDALIDITY change was spurious, if
# it is, just discard it and go on.
with self.conn_pool.get() as crispin_client:
crispin_client.select_folder(self.folder_name, lambda *args: True)
remote_uidvalidity = crispin_client.selected_uidvalidity
remote_uidnext = crispin_client.selected_uidnext
if remote_uidvalidity <= self.uidvalidity:
log.debug('UIDVALIDITY unchanged')
return
# Otherwise, if the UIDVALIDITY really has changed, discard all saved
# UIDs for the folder, mark associated messages for garbage-collection,
# and return to the 'initial' state to resync.
# This will cause message and threads to be deleted and recreated, but
# uidinvalidity is sufficiently rare that this tradeoff is acceptable.
with session_scope(self.namespace_id) as db_session:
invalid_uids = {
uid for uid, in db_session.query(ImapUid.msg_uid).
filter_by(account_id=self.account_id,
folder_id=self.folder_id)
}
common.remove_deleted_uids(self.account_id, self.folder_id,
invalid_uids, db_session)
self.uidvalidity = remote_uidvalidity
self.highestmodseq = None
self.uidnext = remote_uidnext
def test_drafts_deleted_synchronously(db, default_account, thread, message,
imapuid, folder):
message.is_draft = True
msg_uid = imapuid.msg_uid
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
db.session.expire_all()
assert inspect(message).deleted
assert inspect(thread).deleted
def test_drafts_deleted_synchronously(db, default_account, thread, message,
imapuid, folder):
message.is_draft = True
msg_uid = imapuid.msg_uid
remove_deleted_uids(default_account.id, folder.id, [msg_uid], db.session)
db.session.expire_all()
assert inspect(message).deleted
assert inspect(thread).deleted
def test_messages_deleted_asynchronously(db, default_account, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(default_account.id, folder.id,
{msg_uid: GmailFlags((), ('label', ))}, db.session)
assert 'label' in [cat.display_name for cat in message.categories]
remove_deleted_uids(default_account.id, folder.id, [msg_uid], db.session)
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2
# Check that message categories do get updated synchronously.
assert 'label' not in [cat.display_name for cat in message.categories]
def test_messages_deleted_asynchronously(db, default_account, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(default_account.id, db.session, folder.name, folder.id,
[msg_uid], {msg_uid: GmailFlags((), ('label',))})
assert 'label' in [cat.display_name for cat in message.categories]
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2
# Check that message categories do get updated synchronously.
assert 'label' not in [cat.display_name for cat in message.categories]
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
remote_uids = sorted(crispin_client.all_uids(), key=int)
with self.syncmanager_lock:
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id, self.folder_id,
set(local_uids) - set(remote_uids))
unknown_uids = set(remote_uids) - local_uids
with session_scope(self.namespace_id) as db_session:
self.update_uid_counts(
db_session, remote_uid_count=len(remote_uids),
download_uid_count=len(unknown_uids))
change_poller = spawn(self.poll_for_changes)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
if self.is_all_mail(crispin_client):
# Prioritize UIDs for messages in the inbox folder.
if len(remote_uids) < 1e6:
inbox_uids = set(
crispin_client.search_uids(['X-GM-LABELS', 'inbox']))
else:
# The search above is really slow (times out) on really
# large mailboxes, so bound the search to messages within
# the past month in order to get anywhere.
since = datetime.utcnow() - timedelta(days=30)
inbox_uids = set(crispin_client.search_uids([
'X-GM-LABELS', 'inbox',
'SINCE', since]))
uids_to_download = (sorted(unknown_uids - inbox_uids) +
sorted(unknown_uids & inbox_uids))
else:
uids_to_download = sorted(unknown_uids)
for uids in chunk(reversed(uids_to_download), 1024):
g_metadata = crispin_client.g_metadata(uids)
# UIDs might have been expunged since sync started, in which
# case the g_metadata call above will return nothing.
# They may also have been preemptively downloaded by thread
# expansion. We can omit such UIDs.
uids = [u for u in uids if u in g_metadata and u not in
self.saved_uids]
self.batch_download_uids(crispin_client, uids, g_metadata)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def test_only_uids_deleted_synchronously(db, default_account,
default_namespace, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(default_account.id, db.session, folder.name, folder.id,
[msg_uid], {msg_uid: GmailFlags((), ('label', ))})
assert 'label' in [t.name for t in thread.tags]
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 1
# Check that thread tags do get updated synchronously.
assert 'label' not in [t.name for t in thread.tags]
def test_only_uids_deleted_synchronously(db, default_account,
default_namespace, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(default_account.id, db.session, folder.name, folder.id,
[msg_uid], {msg_uid: GmailFlags((), ('label',))})
assert 'label' in [t.name for t in thread.tags]
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2
# Check that thread tags do get updated synchronously.
assert 'label' not in [t.name for t in thread.tags]
def test_drafts_deleted_synchronously(db, default_account, thread, message,
imapuid, folder):
message.is_draft = True
db.session.commit()
msg_uid = imapuid.msg_uid
remove_deleted_uids(default_account.id, folder.id, [msg_uid])
db.session.expire_all()
with pytest.raises(ObjectDeletedError):
message.id
with pytest.raises(ObjectDeletedError):
thread.id
def refresh_flags_impl(self, crispin_client, max_uids):
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(
account_id=self.account_id, session=db_session, folder_id=self.folder_id, limit=max_uids
)
flags = crispin_client.flags(local_uids)
expunged_uids = set(local_uids).difference(flags.keys())
common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids)
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id, flags, db_session)
def test_deletion_deferred_with_longer_ttl(db, default_account,
default_namespace, message, thread,
folder, imapuid):
msg_uid = imapuid.msg_uid
handler = DeleteHandler(account_id=default_account.id,
namespace_id=default_namespace.id,
uid_accessor=lambda m: m.imapuids,
message_ttl=1)
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
handler.check()
# Would raise ObjectDeletedError if objects were deleted
message.id
thread.id
def test_deletion_deferred_with_longer_ttl(db, default_account,
default_namespace, message, thread,
folder, imapuid):
msg_uid = imapuid.msg_uid
handler = DeleteHandler(account_id=default_account.id,
namespace_id=default_namespace.id,
uid_accessor=lambda m: m.imapuids,
message_ttl=1)
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
handler.check()
# Would raise ObjectDeletedError if objects were deleted
message.id
thread.id
def test_deletion_with_short_ttl(db, default_account, default_namespace,
message, thread, folder, imapuid):
msg_uid = imapuid.msg_uid
handler = DeleteHandler(account_id=default_account.id,
namespace_id=default_namespace.id,
uid_accessor=lambda m: m.imapuids,
message_ttl=0)
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
handler.check()
# Check that objects were actually deleted
with pytest.raises(ObjectDeletedError):
message.id
with pytest.raises(ObjectDeletedError):
thread.id
def test_deletion_with_short_ttl(db, default_account, default_namespace,
message, thread, folder, imapuid):
msg_uid = imapuid.msg_uid
handler = DeleteHandler(account_id=default_account.id,
namespace_id=default_namespace.id,
uid_accessor=lambda m: m.imapuids,
message_ttl=0)
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
handler.check()
# Check that objects were actually deleted
with pytest.raises(ObjectDeletedError):
message.id
with pytest.raises(ObjectDeletedError):
thread.id
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id,
self.folder_id,
set(local_uids).difference(remote_uids),
)
new_uids = set(remote_uids).difference(local_uids)
with session_scope(self.namespace_id) as db_session:
account = db_session.query(Account).get(self.account_id)
throttled = account.throttled
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids),
)
change_poller = gevent.spawn(self.poll_for_changes)
bind_context(change_poller, "changepoller", self.account_id,
self.folder_id)
uids = sorted(new_uids, reverse=True)
count = 0
for uid in uids:
# The speedup from batching appears to be less clear for
# non-Gmail accounts, so for now just download one-at-a-time.
self.download_and_commit_uids(crispin_client, [uid])
self.heartbeat_status.publish()
count += 1
if throttled and count >= THROTTLE_COUNT:
# Throttled accounts' folders sync at a rate of
# 1 message/ minute, after the first approx. THROTTLE_COUNT
# messages per folder are synced.
# Note this is an approx. limit since we use the #(uids),
# not the #(messages).
gevent.sleep(THROTTLE_WAIT)
finally:
if change_poller is not None:
# schedule change_poller to die
gevent.kill(change_poller)
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning(
'got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
# Highestmodseq has changed, update accordingly.
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
changed_flags, db_session)
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
# If new UIDs have appeared since we last checked in
# get_new_uids, save them first. We want to always have the
# latest UIDs before expunging anything, in order to properly
# capture draft revisions.
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
if remote_uids and lastseenuid < max(remote_uids):
log.info('Downloading new UIDs before expunging')
self.get_new_uids(crispin_client)
with session_scope(self.namespace_id) as db_session:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
db_session.commit()
self.highestmodseq = new_highestmodseq
def refresh_flags_impl(self, crispin_client, max_uids):
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(account_id=self.account_id,
session=db_session,
folder_id=self.folder_id,
limit=max_uids)
flags = crispin_client.flags(local_uids)
expunged_uids = set(local_uids).difference(flags.keys())
with session_scope(self.namespace_id) as db_session:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
common.update_metadata(self.account_id, self.folder_id, flags,
db_session)
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
# Highestmodseq has changed, update accordingly.
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
changed_flags, db_session)
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
# If new UIDs have appeared since we last checked in
# get_new_uids, save them first. We want to always have the
# latest UIDs before expunging anything, in order to properly
# capture draft revisions.
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
if remote_uids and lastseenuid < max(remote_uids):
log.info('Downloading new UIDs before expunging')
self.get_new_uids(crispin_client)
with session_scope(self.namespace_id) as db_session:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
db_session.commit()
self.highestmodseq = new_highestmodseq
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id, self.folder_id,
set(local_uids).difference(remote_uids),
db_session)
new_uids = set(remote_uids).difference(local_uids)
with session_scope(self.namespace_id) as db_session:
account = db_session.query(Account).get(self.account_id)
throttled = account.throttled
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids))
change_poller = spawn(self.poll_for_changes)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
uids = sorted(new_uids, reverse=True)
count = 0
for uid in uids:
# The speedup from batching appears to be less clear for
# non-Gmail accounts, so for now just download one-at-a-time.
self.download_and_commit_uids(crispin_client, [uid])
self.heartbeat_status.publish()
count += 1
if throttled and count >= THROTTLE_COUNT:
# Throttled accounts' folders sync at a rate of
# 1 message/ minute, after the first approx. THROTTLE_COUNT
# messages per folder are synced.
# Note this is an approx. limit since we use the #(uids),
# not the #(messages).
sleep(THROTTLE_WAIT)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def remove_deleted_uids(self, db_session, local_uids, remote_uids):
""" Remove imapuid entries that no longer exist on the remote.
Works as follows:
1. Do a LIST on the current folder to see what messages are on the
server.
2. Compare to message uids stored locally.
3. Purge uids we have locally but not on the server. Ignore
remote uids that aren't saved locally.
Make SURE to be holding `syncmanager_lock` when calling this function;
we do not grab it here to allow callers to lock higher level
functionality. """
to_delete = set(local_uids) - set(remote_uids)
common.remove_deleted_uids(self.account_id, db_session, to_delete,
self.folder_id)
def remove_deleted_uids(self, db_session, local_uids, remote_uids):
""" Remove imapuid entries that no longer exist on the remote.
Works as follows:
1. Do a LIST on the current folder to see what messages are on the
server.
2. Compare to message uids stored locally.
3. Purge uids we have locally but not on the server. Ignore
remote uids that aren't saved locally.
Make SURE to be holding `syncmanager_lock` when calling this function;
we do not grab it here to allow callers to lock higher level
functionality. """
to_delete = set(local_uids) - set(remote_uids)
common.remove_deleted_uids(self.account_id, db_session, to_delete,
self.folder_id)
def test_messages_deleted_asynchronously(db, default_account, thread, message,
imapuid, folder):
msg_uid = imapuid.msg_uid
update_metadata(
default_account.id,
folder.id,
folder.canonical_name,
{msg_uid: GmailFlags((), ("label", ), None)},
db.session,
)
assert "label" in [cat.display_name for cat in message.categories]
remove_deleted_uids(default_account.id, folder.id, [msg_uid])
db.session.expire_all()
assert abs((message.deleted_at - datetime.utcnow()).total_seconds()) < 2
# Check that message categories do get updated synchronously.
assert "label" not in [cat.display_name for cat in message.categories]
def test_threads_only_deleted_when_no_messages_left(db, default_account,
default_namespace, message,
thread, folder, imapuid):
msg_uid = imapuid.msg_uid
handler = DeleteHandler(account_id=default_account.id,
namespace_id=default_namespace.id,
uid_accessor=lambda m: m.imapuids,
message_ttl=0)
# Add another message onto the thread
add_fake_message(db.session, default_namespace.id, thread)
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
handler.check()
# Check that the orphaned message was deleted.
with pytest.raises(ObjectDeletedError):
message.id
# Would raise ObjectDeletedError if thread was deleted.
thread.id
def test_threads_only_deleted_when_no_messages_left(db, default_account,
default_namespace, message,
thread, folder, imapuid):
msg_uid = imapuid.msg_uid
handler = DeleteHandler(account_id=default_account.id,
namespace_id=default_namespace.id,
uid_accessor=lambda m: m.imapuids,
message_ttl=0)
# Add another message onto the thread
add_fake_message(db.session, default_namespace.id, thread)
remove_deleted_uids(default_account.id, db.session, [msg_uid], folder.id)
handler.check()
# Check that the orphaned message was deleted.
with pytest.raises(ObjectDeletedError):
message.id
# Would raise ObjectDeletedError if thread was deleted.
thread.id
def refresh_flags_impl(self, crispin_client, max_uids):
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
# Check for any deleted messages.
remote_uids = crispin_client.all_uids()
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
with self.syncmanager_lock:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids)
# Get recent UIDs to monitor for flag changes.
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(
account_id=self.account_id,
session=db_session,
folder_id=self.folder_id,
limit=max_uids,
)
flags = crispin_client.flags(local_uids)
if max_uids in self.flags_fetch_results and self.flags_fetch_results[
max_uids] == (local_uids, flags):
# If the flags fetch response is exactly the same as the last one
# we got, then we don't need to persist any changes.
# Stopped logging this to reduce overall logging volume
# log.debug('Unchanged flags refresh response, '
# 'not persisting changes', max_uids=max_uids)
return
log.debug("Changed flags refresh response, persisting changes",
max_uids=max_uids)
expunged_uids = set(local_uids).difference(flags.keys())
with self.syncmanager_lock:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids)
with self.syncmanager_lock, session_scope(
self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
self.folder_role, flags, db_session)
self.flags_fetch_results[max_uids] = (local_uids, flags)
def test_deleting_from_a_message_with_multiple_uids(db, default_account,
message, thread):
"""Check that deleting a imapuid from a message with
multiple uids doesn't mark the message for deletion."""
inbox_folder = default_account.inbox_folder
sent_folder = default_account.sent_folder
add_fake_imapuid(db.session, default_account.id, message, sent_folder,
1337)
add_fake_imapuid(db.session, default_account.id, message, inbox_folder,
2222)
assert len(message.imapuids) == 2
remove_deleted_uids(default_account.id, db.session, [2222],
inbox_folder.id)
assert message.deleted_at is None, \
"The associated message should not have been marked for deletion."
assert len(message.imapuids) == 1, \
"The message should have only one imapuid."
def test_deleting_from_a_message_with_multiple_uids(db, default_account,
message, thread):
"""Check that deleting a imapuid from a message with
multiple uids doesn't mark the message for deletion."""
inbox_folder = default_account.inbox_folder
sent_folder = default_account.sent_folder
add_fake_imapuid(db.session, default_account.id, message, sent_folder,
1337)
add_fake_imapuid(db.session, default_account.id, message, inbox_folder,
2222)
assert len(message.imapuids) == 2
remove_deleted_uids(default_account.id, db.session, [2222],
inbox_folder.id)
assert message.deleted_at is None, \
"The associated message should not have been marked for deletion."
assert len(message.imapuids) == 1, \
"The message should have only one imapuid."
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with session_scope() as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id, self.folder_id,
set(local_uids).difference(remote_uids),
db_session)
new_uids = set(remote_uids).difference(local_uids)
with session_scope() as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids))
change_poller = spawn(self.poll_for_changes)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
uids = sorted(new_uids, reverse=True)
for uid in uids:
# The speedup from batching appears to be less clear for
# non-Gmail accounts, so for now just download one-at-a-time.
self.download_and_commit_uids(crispin_client, [uid])
self.heartbeat_status.publish()
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the change_poller greenlet
# needs to be killed when this greenlet is interrupted
change_poller = None
try:
assert crispin_client.selected_folder_name == self.folder_name
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with session_scope() as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
common.remove_deleted_uids(
self.account_id, self.folder_id,
set(local_uids).difference(remote_uids), db_session)
new_uids = set(remote_uids).difference(local_uids)
with session_scope() as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
# This is the initial size of our download_queue
download_uid_count=len(new_uids))
change_poller = spawn(self.poll_for_changes)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
uids = sorted(new_uids, reverse=True)
for uid in uids:
# The speedup from batching appears to be less clear for
# non-Gmail accounts, so for now just download one-at-a-time.
self.download_and_commit_uids(crispin_client, [uid])
self.heartbeat_status.publish()
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
log.info('HIGHESTMODSEQ has changed, getting changed UIDs',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
# In order to be able to sync changes to tens of thousands of flags at
# once, we commit updates in batches. We do this in ascending order by
# modseq and periodically "checkpoint" our saved highestmodseq. (It's
# safe to checkpoint *because* we go in ascending order by modseq.)
# That way if the process gets restarted halfway through this refresh,
# we don't have to completely start over. It's also slow to load many
# objects into the SQLAlchemy session and then issue lots of commits;
# we avoid that by batching.
flag_batches = chunk(
sorted(changed_flags.items(), key=lambda (k, v): v.modseq),
CONDSTORE_FLAGS_REFRESH_BATCH_SIZE)
for flag_batch in flag_batches:
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
self.folder_role, dict(flag_batch),
db_session)
if len(flag_batch) == CONDSTORE_FLAGS_REFRESH_BATCH_SIZE:
interim_highestmodseq = max(v.modseq for k, v in flag_batch)
self.highestmodseq = interim_highestmodseq
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
# If new UIDs have appeared since we last checked in
# get_new_uids, save them first. We want to always have the
# latest UIDs before expunging anything, in order to properly
# capture draft revisions.
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
if remote_uids and lastseenuid < max(remote_uids):
log.info('Downloading new UIDs before expunging')
self.get_new_uids(crispin_client)
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids)
self.highestmodseq = new_highestmodseq