def check_uid_changes(self, crispin_client, download_stack,
async_download):
crispin_client.select_folder(self.folder_name, uidvalidity_cb)
new_highestmodseq = crispin_client.selected_highestmodseq
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(
self.account_id, db_session, self.folder_name)
# Ensure that we have an initial highestmodseq value stored before
# we begin polling for changes.
if saved_folder_info is None or \
saved_folder_info.highestmodseq is None:
assert (crispin_client.selected_uidvalidity is not None
and crispin_client.selected_highestmodseq is
not None)
saved_folder_info = common.update_folder_info(
crispin_client.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
saved_highestmodseq = saved_folder_info.highestmodseq
if new_highestmodseq == saved_highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < saved_highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=saved_highestmodseq)
return
save_folder_names(log, self.account_id,
crispin_client.folder_names(), db_session)
# Highestmodseq has changed, update accordingly.
new_uidvalidity = crispin_client.selected_uidvalidity
changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
remote_uids = crispin_client.all_uids()
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
stack_uids = {uid for uid, _ in download_stack}
local_with_pending_uids = local_uids | stack_uids
new, updated = new_or_updated(changed_uids, local_with_pending_uids)
if changed_uids:
log.info("Changed UIDs", message="new: {} updated: {}"
.format(len(new), len(updated)),
new_uid_count=len(new), updated_uid_count=len(updated))
self.update_metadata(crispin_client, updated)
self.highestmodseq_callback(crispin_client, new, updated,
download_stack, async_download)
with mailsync_session_scope() as db_session:
with self.syncmanager_lock:
self.remove_deleted_uids(db_session, local_uids, remote_uids)
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def __fetch_g_metadata(self, crispin_client, uids):
assert self.folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
remote_g_metadata = None
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(
self.account_id, db_session, self.folder_name)
saved_highestmodseq = or_none(saved_folder_info, lambda i:
i.highestmodseq)
if saved_highestmodseq is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata = self.__retrieve_saved_g_metadata(
crispin_client, uids, saved_highestmodseq)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(remote_g_metadata_cache_file(self.account_id,
self.folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
with mailsync_session_scope() as db_session:
common.update_folder_info(self.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata
def check_uid_changes(self, crispin_client, download_stack,
async_download):
crispin_client.select_folder(self.folder_name, uidvalidity_cb)
new_highestmodseq = crispin_client.selected_highestmodseq
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
# Ensure that we have an initial highestmodseq value stored before
# we begin polling for changes.
if saved_folder_info is None or \
saved_folder_info.highestmodseq is None:
assert (crispin_client.selected_uidvalidity is not None
and crispin_client.selected_highestmodseq is not None)
saved_folder_info = common.update_folder_info(
crispin_client.account_id, db_session, self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
saved_highestmodseq = saved_folder_info.highestmodseq
if new_highestmodseq == saved_highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < saved_highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning(
'got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=saved_highestmodseq)
return
# Highestmodseq has changed, update accordingly.
new_uidvalidity = crispin_client.selected_uidvalidity
changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
remote_uids = crispin_client.all_uids()
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
stack_uids = set(download_stack.keys())
local_with_pending_uids = local_uids | stack_uids
new, updated = new_or_updated(changed_uids, local_with_pending_uids)
if changed_uids:
log.info("Changed UIDs",
message="new: {} updated: {}".format(
len(new), len(updated)),
new_uid_count=len(new),
updated_uid_count=len(updated))
self.update_metadata(crispin_client, updated)
self.highestmodseq_callback(crispin_client, new, updated,
download_stack, async_download)
with mailsync_session_scope() as db_session:
with self.syncmanager_lock:
self.remove_deleted_uids(db_session, local_uids, remote_uids)
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def __fetch_g_metadata(self, crispin_client, uids):
assert self.folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
remote_g_metadata = None
update_uid_count = 0
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
saved_highestmodseq = or_none(saved_folder_info,
lambda i: i.highestmodseq)
if saved_highestmodseq is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata, update_uid_count = \
self.__retrieve_saved_g_metadata(crispin_client, uids,
saved_highestmodseq)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(
remote_g_metadata_cache_file(self.account_id,
self.folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
with mailsync_session_scope() as db_session:
common.update_folder_info(self.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata, update_uid_count
def poll_impl(self, crispin_client):
log.bind(state='poll')
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(
crispin_client.account_id, db_session, self.folder_name)
saved_highestmodseq = saved_folder_info.highestmodseq
# Start a session since we're going to IDLE below anyway...
# This also resets the folder name cache, which we want in order to
# detect folder/label additions and deletions.
status = crispin_client.select_folder(
self.folder_name, uidvalidity_cb(crispin_client.account_id))
log.debug(current_modseq=status['HIGHESTMODSEQ'],
saved_modseq=saved_highestmodseq)
if status['HIGHESTMODSEQ'] > saved_highestmodseq:
with mailsync_session_scope() as db_session:
acc = db_session.query(ImapAccount).get(self.account_id)
save_folder_names(log, acc, crispin_client.folder_names(),
db_session)
self.highestmodseq_update(crispin_client, saved_highestmodseq)
# We really only want to idle on a folder for new messages. Idling on
# `All Mail` won't tell us when messages are archived from the Inbox
if self.folder_name.lower() in IDLE_FOLDERS:
status = crispin_client.select_folder(
self.folder_name, uidvalidity_cb(crispin_client.account_id))
# Idle doesn't pick up flag changes, so we don't want to idle for
# very long, or we won't detect things like messages being marked
# as read.
idle_frequency = 30
log.info('idling', timeout=idle_frequency)
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=idle_frequency)
# If we want to do something with the response, but lousy
# because it uses sequence IDs instead of UIDs
# resp = c.idle_check(timeout=shared_state['poll_frequency'])
# r = dict( EXISTS=[], EXPUNGE=[])
# for msg_uid, cmd in resp:
# r[cmd].append(msg_uid)
# print r
crispin_client.conn.idle_done()
log.info('IDLE triggered poll')
else:
log.info('IDLE sleeping', seconds=self.poll_frequency)
sleep(self.poll_frequency)
def uidvalidity_cb(account_id, folder_name, select_info):
assert folder_name is not None and select_info is not None, "must start IMAP session before verifying UIDVALIDITY"
with session_scope(account_id) as db_session:
saved_folder_info = common.get_folder_info(account_id, db_session, folder_name)
saved_uidvalidity = or_none(saved_folder_info, lambda i: i.uidvalidity)
selected_uidvalidity = select_info["UIDVALIDITY"]
if saved_folder_info:
is_valid = saved_uidvalidity is None or selected_uidvalidity <= saved_uidvalidity
if not is_valid:
raise UidInvalid(
"folder: {}, remote uidvalidity: {}, "
"cached uidvalidity: {}".format(folder_name.encode("utf-8"), selected_uidvalidity, saved_uidvalidity)
)
return select_info
def uidvalidity_cb(account_id, folder_name, select_info):
assert folder_name is not None and select_info is not None, \
"must start IMAP session before verifying UIDVALIDITY"
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(account_id, db_session,
folder_name)
saved_uidvalidity = or_none(saved_folder_info, lambda i: i.uidvalidity)
selected_uidvalidity = select_info['UIDVALIDITY']
if saved_folder_info:
is_valid = common.uidvalidity_valid(account_id, selected_uidvalidity,
folder_name, saved_uidvalidity)
if not is_valid:
raise UidInvalid('folder: {}, remote uidvalidity: {}, '
'cached uidvalidity: {}'.format(
folder_name.encode('utf-8'),
selected_uidvalidity, saved_uidvalidity))
return select_info
def uidvalidity_cb(account_id, folder_name, select_info):
assert (folder_name is not None and select_info is not None
), "must start IMAP session before verifying UIDVALIDITY"
with session_scope(account_id) as db_session:
saved_folder_info = common.get_folder_info(account_id, db_session,
folder_name)
saved_uidvalidity = or_none(saved_folder_info, lambda i: i.uidvalidity)
selected_uidvalidity = select_info["UIDVALIDITY"]
if saved_folder_info:
is_valid = (saved_uidvalidity is None
or selected_uidvalidity <= saved_uidvalidity)
if not is_valid:
raise UidInvalid("folder: {}, remote uidvalidity: {}, "
"cached uidvalidity: {}".format(
folder_name.encode("utf-8"),
selected_uidvalidity, saved_uidvalidity))
return select_info
def __check_flags(self, crispin_client, db_session, local_uids):
""" Update message flags if folder has changed on the remote.
If we have saved validity info for this folder, make sure the folder
hasn't changed since we saved it. Otherwise we need to query for flag
changes too.
"""
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
if saved_folder_info is not None:
last_highestmodseq = saved_folder_info.highestmodseq
if last_highestmodseq > crispin_client.selected_highestmodseq:
uids = crispin_client.new_and_updated_uids(last_highestmodseq)
if uids:
_, updated = new_or_updated(uids, local_uids)
self.update_metadata(crispin_client, updated)
def uidvalidity_cb(account_id, folder_name, select_info):
assert folder_name is not None and select_info is not None, \
"must start IMAP session before verifying UIDVALIDITY"
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(account_id, db_session,
folder_name)
saved_uidvalidity = or_none(saved_folder_info, lambda i:
i.uidvalidity)
selected_uidvalidity = select_info['UIDVALIDITY']
if saved_folder_info:
is_valid = common.uidvalidity_valid(account_id,
selected_uidvalidity,
folder_name, saved_uidvalidity)
if not is_valid:
raise UidInvalid(
'folder: {}, remote uidvalidity: {}, '
'cached uidvalidity: {}'.format(
folder_name, selected_uidvalidity, saved_uidvalidity))
return select_info
def initial_sync_impl(self, crispin_client, local_uids,
uid_download_stack):
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
if saved_folder_info is None:
assert (crispin_client.selected_uidvalidity is not None and
crispin_client.selected_highestmodseq is not None)
common.update_folder_info(
crispin_client.account_id, db_session, self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
self.__check_flags(crispin_client, db_session, local_uids)
return FolderSyncEngine.initial_sync_impl(
self, crispin_client, local_uids, uid_download_stack,
spawn_flags_refresh_poller=False)