def get_new_uids(self, crispin_client):
try:
remote_uidnext = crispin_client.conn.folder_status(
self.folder_name, ['UIDNEXT']).get('UIDNEXT')
except ValueError:
# Work around issue where ValueError is raised on parsing STATUS
# response.
log.warning('Error getting UIDNEXT', exc_info=True)
remote_uidnext = None
except imaplib.IMAP4.error as e:
if '[NONEXISTENT]' in e.message:
raise FolderMissingError()
else:
raise e
if remote_uidnext is not None and remote_uidnext == self.uidnext:
return
log.info('UIDNEXT changed, checking for new UIDs',
remote_uidnext=remote_uidnext, saved_uidnext=self.uidnext)
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
latest_uids = crispin_client.conn.fetch('{}:*'.format(lastseenuid + 1),
['UID']).keys()
new_uids = set(latest_uids) - {lastseenuid}
if new_uids:
for uid in sorted(new_uids):
self.download_and_commit_uids(crispin_client, [uid])
self.uidnext = remote_uidnext
def get_new_uids(self, crispin_client):
try:
remote_uidnext = crispin_client.conn.folder_status(
self.folder_name, ['UIDNEXT']).get('UIDNEXT')
except ValueError:
# Work around issue where ValueError is raised on parsing STATUS
# response.
log.warning('Error getting UIDNEXT', exc_info=True)
remote_uidnext = None
except imaplib.IMAP4.error as e:
if '[NONEXISTENT]' in e.message:
raise FolderMissingError()
else:
raise e
if remote_uidnext is not None and remote_uidnext == self.uidnext:
return
log.info('UIDNEXT changed, checking for new UIDs',
remote_uidnext=remote_uidnext,
saved_uidnext=self.uidnext)
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
latest_uids = crispin_client.conn.fetch('{}:*'.format(lastseenuid + 1),
['UID']).keys()
new_uids = set(latest_uids) - {lastseenuid}
if new_uids:
for uid in sorted(new_uids):
self.download_and_commit_uids(crispin_client, [uid])
self.uidnext = remote_uidnext
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning(
'got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
# Highestmodseq has changed, update accordingly.
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
changed_flags, db_session)
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
# If new UIDs have appeared since we last checked in
# get_new_uids, save them first. We want to always have the
# latest UIDs before expunging anything, in order to properly
# capture draft revisions.
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
if remote_uids and lastseenuid < max(remote_uids):
log.info('Downloading new UIDs before expunging')
self.get_new_uids(crispin_client)
with session_scope(self.namespace_id) as db_session:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
db_session.commit()
self.highestmodseq = new_highestmodseq
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
# Highestmodseq has changed, update accordingly.
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
changed_flags, db_session)
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
# If new UIDs have appeared since we last checked in
# get_new_uids, save them first. We want to always have the
# latest UIDs before expunging anything, in order to properly
# capture draft revisions.
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
if remote_uids and lastseenuid < max(remote_uids):
log.info('Downloading new UIDs before expunging')
self.get_new_uids(crispin_client)
with session_scope(self.namespace_id) as db_session:
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids, db_session)
db_session.commit()
self.highestmodseq = new_highestmodseq
def get_new_uids(self, crispin_client):
try:
remote_uidnext = crispin_client.conn.folder_status(
self.folder_name, ["UIDNEXT"]).get(b"UIDNEXT")
except ValueError:
# Work around issue where ValueError is raised on parsing STATUS
# response.
log.warning("Error getting UIDNEXT", exc_info=True)
remote_uidnext = None
except imaplib.IMAP4.error as e:
# TODO: match with CrispinClient.select_folder
message = e.args[0] if e.args else ""
if ("[NONEXISTENT]" in message or "does not exist" in message
or "doesn't exist" in message):
raise FolderMissingError()
else:
raise e
if remote_uidnext is not None and remote_uidnext == self.uidnext:
return
log.debug(
"UIDNEXT changed, checking for new UIDs",
remote_uidnext=remote_uidnext,
saved_uidnext=self.uidnext,
)
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
latest_uids = crispin_client.conn.fetch("{}:*".format(lastseenuid + 1),
["UID"]).keys()
new_uids = set(latest_uids) - {lastseenuid}
if new_uids:
for uid in sorted(new_uids):
self.download_and_commit_uids(crispin_client, [uid])
self.uidnext = remote_uidnext
def condstore_refresh_flags(self, crispin_client):
new_highestmodseq = crispin_client.conn.folder_status(
self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
# Ensure that we have an initial highestmodseq value stored before we
# begin polling for changes.
if self.highestmodseq is None:
self.highestmodseq = new_highestmodseq
if new_highestmodseq == self.highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < self.highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
return
log.info('HIGHESTMODSEQ has changed, getting changed UIDs',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=self.highestmodseq)
crispin_client.select_folder(self.folder_name, self.uidvalidity_cb)
changed_flags = crispin_client.condstore_changed_flags(
self.highestmodseq)
remote_uids = crispin_client.all_uids()
# In order to be able to sync changes to tens of thousands of flags at
# once, we commit updates in batches. We do this in ascending order by
# modseq and periodically "checkpoint" our saved highestmodseq. (It's
# safe to checkpoint *because* we go in ascending order by modseq.)
# That way if the process gets restarted halfway through this refresh,
# we don't have to completely start over. It's also slow to load many
# objects into the SQLAlchemy session and then issue lots of commits;
# we avoid that by batching.
flag_batches = chunk(
sorted(changed_flags.items(), key=lambda (k, v): v.modseq),
CONDSTORE_FLAGS_REFRESH_BATCH_SIZE)
for flag_batch in flag_batches:
with session_scope(self.namespace_id) as db_session:
common.update_metadata(self.account_id, self.folder_id,
self.folder_role, dict(flag_batch),
db_session)
if len(flag_batch) == CONDSTORE_FLAGS_REFRESH_BATCH_SIZE:
interim_highestmodseq = max(v.modseq for k, v in flag_batch)
self.highestmodseq = interim_highestmodseq
with session_scope(self.namespace_id) as db_session:
local_uids = common.local_uids(self.account_id, db_session,
self.folder_id)
expunged_uids = set(local_uids).difference(remote_uids)
if expunged_uids:
# If new UIDs have appeared since we last checked in
# get_new_uids, save them first. We want to always have the
# latest UIDs before expunging anything, in order to properly
# capture draft revisions.
with session_scope(self.namespace_id) as db_session:
lastseenuid = common.lastseenuid(self.account_id, db_session,
self.folder_id)
if remote_uids and lastseenuid < max(remote_uids):
log.info('Downloading new UIDs before expunging')
self.get_new_uids(crispin_client)
common.remove_deleted_uids(self.account_id, self.folder_id,
expunged_uids)
self.highestmodseq = new_highestmodseq