def check_uid_changes(self, crispin_client, download_stack,
async_download):
crispin_client.select_folder(self.folder_name, uidvalidity_cb)
new_highestmodseq = crispin_client.selected_highestmodseq
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
# Ensure that we have an initial highestmodseq value stored before
# we begin polling for changes.
if saved_folder_info is None or \
saved_folder_info.highestmodseq is None:
assert (crispin_client.selected_uidvalidity is not None
and crispin_client.selected_highestmodseq is not None)
saved_folder_info = common.update_folder_info(
crispin_client.account_id, db_session, self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
saved_highestmodseq = saved_folder_info.highestmodseq
if new_highestmodseq == saved_highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < saved_highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning(
'got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=saved_highestmodseq)
return
# Highestmodseq has changed, update accordingly.
new_uidvalidity = crispin_client.selected_uidvalidity
changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
remote_uids = crispin_client.all_uids()
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
stack_uids = set(download_stack.keys())
local_with_pending_uids = local_uids | stack_uids
new, updated = new_or_updated(changed_uids, local_with_pending_uids)
if changed_uids:
log.info("Changed UIDs",
message="new: {} updated: {}".format(
len(new), len(updated)),
new_uid_count=len(new),
updated_uid_count=len(updated))
self.update_metadata(crispin_client, updated)
self.highestmodseq_callback(crispin_client, new, updated,
download_stack, async_download)
with mailsync_session_scope() as db_session:
with self.syncmanager_lock:
self.remove_deleted_uids(db_session, local_uids, remote_uids)
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def __fetch_g_metadata(self, crispin_client, uids):
assert self.folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
remote_g_metadata = None
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(
self.account_id, db_session, self.folder_name)
saved_highestmodseq = or_none(saved_folder_info, lambda i:
i.highestmodseq)
if saved_highestmodseq is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata = self.__retrieve_saved_g_metadata(
crispin_client, uids, saved_highestmodseq)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(remote_g_metadata_cache_file(self.account_id,
self.folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
with mailsync_session_scope() as db_session:
common.update_folder_info(self.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata
def check_uid_changes(self, crispin_client, download_stack,
async_download):
crispin_client.select_folder(self.folder_name, uidvalidity_cb)
new_highestmodseq = crispin_client.selected_highestmodseq
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(
self.account_id, db_session, self.folder_name)
# Ensure that we have an initial highestmodseq value stored before
# we begin polling for changes.
if saved_folder_info is None or \
saved_folder_info.highestmodseq is None:
assert (crispin_client.selected_uidvalidity is not None
and crispin_client.selected_highestmodseq is
not None)
saved_folder_info = common.update_folder_info(
crispin_client.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
saved_highestmodseq = saved_folder_info.highestmodseq
if new_highestmodseq == saved_highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < saved_highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=saved_highestmodseq)
return
save_folder_names(log, self.account_id,
crispin_client.folder_names(), db_session)
# Highestmodseq has changed, update accordingly.
new_uidvalidity = crispin_client.selected_uidvalidity
changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
remote_uids = crispin_client.all_uids()
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
stack_uids = {uid for uid, _ in download_stack}
local_with_pending_uids = local_uids | stack_uids
new, updated = new_or_updated(changed_uids, local_with_pending_uids)
if changed_uids:
log.info("Changed UIDs", message="new: {} updated: {}"
.format(len(new), len(updated)),
new_uid_count=len(new), updated_uid_count=len(updated))
self.update_metadata(crispin_client, updated)
self.highestmodseq_callback(crispin_client, new, updated,
download_stack, async_download)
with mailsync_session_scope() as db_session:
with self.syncmanager_lock:
self.remove_deleted_uids(db_session, local_uids, remote_uids)
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def __fetch_g_metadata(self, crispin_client, uids):
assert self.folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
remote_g_metadata = None
update_uid_count = 0
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
saved_highestmodseq = or_none(saved_folder_info,
lambda i: i.highestmodseq)
if saved_highestmodseq is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata, update_uid_count = \
self.__retrieve_saved_g_metadata(crispin_client, uids,
saved_highestmodseq)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(
remote_g_metadata_cache_file(self.account_id,
self.folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
with mailsync_session_scope() as db_session:
common.update_folder_info(self.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata, update_uid_count
def check_uid_changes(self, crispin_client, download_stack,
async_download):
remote_uids = set(crispin_client.all_uids())
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
# Download new UIDs.
stack_uids = set(download_stack.keys())
local_with_pending_uids = local_uids | stack_uids
for uid in sorted(remote_uids):
if uid not in local_with_pending_uids:
download_stack.put(uid, None)
self.remove_deleted_uids(db_session, local_uids, remote_uids)
if not async_download:
self.download_uids(crispin_client, download_stack)
with mailsync_session_scope() as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
download_uid_count=len(download_stack))
to_refresh = sorted(remote_uids &
local_uids)[-self.refresh_flags_max:]
self.update_metadata(crispin_client, to_refresh)
with mailsync_session_scope() as db_session:
common.update_folder_info(self.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
None,
crispin_client.selected_uidnext)
def highestmodseq_update(self, crispin_client, last_highestmodseq):
new_highestmodseq = crispin_client.selected_highestmodseq
new_uidvalidity = crispin_client.selected_uidvalidity
log.info('starting highestmodseq update',
current_highestmodseq=new_highestmodseq)
changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
remote_uids = crispin_client.all_uids()
local_uids = None
if changed_uids:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
new, updated = new_or_updated(changed_uids, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
local_uids.update(new)
with self.syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
with mailsync_session_scope() as db_session:
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
local_uids = local_uids - deleted_uids
self.update_metadata(crispin_client, updated)
with mailsync_session_scope() as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
download_uid_count=len(new),
update_uid_count=len(updated),
delete_uid_count=len(deleted_uids))
self.highestmodseq_callback(crispin_client, new, updated)
else:
log.info("No new or updated messages")
with mailsync_session_scope() as db_session:
with self.syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
if local_uids is None:
local_uids = common.all_uids(
self.account_id, db_session, self.folder_name)
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids),
delete_uid_count=len(deleted_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def initial_sync_impl(self, crispin_client, local_uids,
uid_download_stack):
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
if saved_folder_info is None:
assert (crispin_client.selected_uidvalidity is not None and
crispin_client.selected_highestmodseq is not None)
common.update_folder_info(
crispin_client.account_id, db_session, self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
self.__check_flags(crispin_client, db_session, local_uids)
return FolderSyncEngine.initial_sync_impl(
self, crispin_client, local_uids, uid_download_stack,
spawn_flags_refresh_poller=False)
