def check_uid_changes(self, crispin_client, download_stack,
async_download):
crispin_client.select_folder(self.folder_name, uidvalidity_cb)
new_highestmodseq = crispin_client.selected_highestmodseq
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
# Ensure that we have an initial highestmodseq value stored before
# we begin polling for changes.
if saved_folder_info is None or \
saved_folder_info.highestmodseq is None:
assert (crispin_client.selected_uidvalidity is not None
and crispin_client.selected_highestmodseq is not None)
saved_folder_info = common.update_folder_info(
crispin_client.account_id, db_session, self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
saved_highestmodseq = saved_folder_info.highestmodseq
if new_highestmodseq == saved_highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < saved_highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning(
'got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=saved_highestmodseq)
return
# Highestmodseq has changed, update accordingly.
new_uidvalidity = crispin_client.selected_uidvalidity
changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
remote_uids = crispin_client.all_uids()
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
stack_uids = set(download_stack.keys())
local_with_pending_uids = local_uids | stack_uids
new, updated = new_or_updated(changed_uids, local_with_pending_uids)
if changed_uids:
log.info("Changed UIDs",
message="new: {} updated: {}".format(
len(new), len(updated)),
new_uid_count=len(new),
updated_uid_count=len(updated))
self.update_metadata(crispin_client, updated)
self.highestmodseq_callback(crispin_client, new, updated,
download_stack, async_download)
with mailsync_session_scope() as db_session:
with self.syncmanager_lock:
self.remove_deleted_uids(db_session, local_uids, remote_uids)
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def check_uid_changes(self, crispin_client, download_stack,
async_download):
crispin_client.select_folder(self.folder_name, uidvalidity_cb)
new_highestmodseq = crispin_client.selected_highestmodseq
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(
self.account_id, db_session, self.folder_name)
# Ensure that we have an initial highestmodseq value stored before
# we begin polling for changes.
if saved_folder_info is None or \
saved_folder_info.highestmodseq is None:
assert (crispin_client.selected_uidvalidity is not None
and crispin_client.selected_highestmodseq is
not None)
saved_folder_info = common.update_folder_info(
crispin_client.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
saved_highestmodseq = saved_folder_info.highestmodseq
if new_highestmodseq == saved_highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < saved_highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=saved_highestmodseq)
return
save_folder_names(log, self.account_id,
crispin_client.folder_names(), db_session)
# Highestmodseq has changed, update accordingly.
new_uidvalidity = crispin_client.selected_uidvalidity
changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
remote_uids = crispin_client.all_uids()
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
stack_uids = {uid for uid, _ in download_stack}
local_with_pending_uids = local_uids | stack_uids
new, updated = new_or_updated(changed_uids, local_with_pending_uids)
if changed_uids:
log.info("Changed UIDs", message="new: {} updated: {}"
.format(len(new), len(updated)),
new_uid_count=len(new), updated_uid_count=len(updated))
self.update_metadata(crispin_client, updated)
self.highestmodseq_callback(crispin_client, new, updated,
download_stack, async_download)
with mailsync_session_scope() as db_session:
with self.syncmanager_lock:
self.remove_deleted_uids(db_session, local_uids, remote_uids)
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def highestmodseq_update(crispin_client, log, folder_name, last_highestmodseq,
highestmodseq_fn, syncmanager_lock):
account_id = crispin_client.account_id
new_highestmodseq = crispin_client.selected_highestmodseq
new_uidvalidity = crispin_client.selected_uidvalidity
log.info('starting highestmodseq update',
current_highestmodseq=new_highestmodseq)
changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
remote_uids = crispin_client.all_uids()
local_uids = None
if changed_uids:
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = account.all_uids(account_id, db_session, folder_name)
new, updated = new_or_updated(changed_uids, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
local_uids += new
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
deleted_uids = remove_deleted_uids(account_id, db_session, log,
folder_name, local_uids,
remote_uids)
local_uids = set(local_uids) - deleted_uids
update_metadata(crispin_client, log, folder_name, updated,
syncmanager_lock)
with session_scope(ignore_soft_deletes=False) as db_session:
update_uid_counts(db_session, log, account_id, folder_name,
remote_uid_count=len(remote_uids),
download_uid_count=len(new),
update_uid_count=len(updated),
delete_uid_count=len(deleted_uids))
highestmodseq_fn(crispin_client, log, folder_name, new,
updated, syncmanager_lock)
else:
log.info("No new or updated messages")
with session_scope(ignore_soft_deletes=False) as db_session:
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
if local_uids is None:
local_uids = account.all_uids(
account_id, db_session, folder_name)
deleted_uids = remove_deleted_uids(crispin_client.account_id,
db_session, log, folder_name,
local_uids, remote_uids)
update_uid_counts(db_session, log, account_id, folder_name,
remote_uid_count=len(remote_uids),
delete_uid_count=len(deleted_uids))
account.update_folder_info(account_id, db_session, folder_name,
new_uidvalidity, new_highestmodseq)
db_session.commit()
def highestmodseq_update(self, crispin_client, last_highestmodseq):
new_highestmodseq = crispin_client.selected_highestmodseq
new_uidvalidity = crispin_client.selected_uidvalidity
log.info('starting highestmodseq update',
current_highestmodseq=new_highestmodseq)
changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
remote_uids = crispin_client.all_uids()
local_uids = None
if changed_uids:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
new, updated = new_or_updated(changed_uids, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
local_uids.update(new)
with self.syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
with mailsync_session_scope() as db_session:
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
local_uids = local_uids - deleted_uids
self.update_metadata(crispin_client, updated)
with mailsync_session_scope() as db_session:
self.update_uid_counts(
db_session,
remote_uid_count=len(remote_uids),
download_uid_count=len(new),
update_uid_count=len(updated),
delete_uid_count=len(deleted_uids))
self.highestmodseq_callback(crispin_client, new, updated)
else:
log.info("No new or updated messages")
with mailsync_session_scope() as db_session:
with self.syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
if local_uids is None:
local_uids = common.all_uids(
self.account_id, db_session, self.folder_name)
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids),
delete_uid_count=len(deleted_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def __update_saved_g_metadata(self, crispin_client, remote_g_metadata,
local_uids):
"""
If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache,
we need to query for any changes since then and update the saved
data.
"""
log.info('Updating cache with latest changes')
# Any uids we don't already have will be downloaded correctly as usual,
# but updated uids need to be updated manually.
# XXX it may actually be faster to just query for X-GM-MSGID for the
# whole folder rather than getting changed UIDs first; MODSEQ queries
# are slow on large folders.
modified = crispin_client.new_and_updated_uids(
crispin_client.selected_highestmodseq)
log.info(modified_msg_count=len(modified))
new, updated = new_or_updated(modified, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
if new:
remote_g_metadata.update(crispin_client.g_metadata(new))
log.info('Updated cache with new messages')
else:
log.info('No new messages to update metadata for')
# Filter out messages that have disappeared.
old_len = len(remote_g_metadata)
current_remote_uids = set(crispin_client.all_uids())
remote_g_metadata = dict((uid, md) for uid, md in
remote_g_metadata.iteritems() if uid in
current_remote_uids)
num_removed = old_len - len(remote_g_metadata)
if num_removed > 0:
log.info(removed_msg_count=num_removed)
set_cache(remote_g_metadata_cache_file(self.account_id,
self.folder_name),
remote_g_metadata)
if updated:
# It's easy and fast to just update these here and now.
# Bigger chunk because the data being fetched here is very small.
for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE):
self.update_metadata(crispin_client, uids)
log.info('updated metadata for modified messages',
msg_count=len(updated))
return len(updated)
else:
log.info('No modified messages to update metadata for')
return 0
def __update_saved_g_metadata(self, crispin_client, remote_g_metadata,
local_uids):
"""
If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache,
we need to query for any changes since then and update the saved
data.
"""
log.info('Updating cache with latest changes')
# Any uids we don't already have will be downloaded correctly as usual,
# but updated uids need to be updated manually.
# XXX it may actually be faster to just query for X-GM-MSGID for the
# whole folder rather than getting changed UIDs first; MODSEQ queries
# are slow on large folders.
modified = crispin_client.new_and_updated_uids(
crispin_client.selected_highestmodseq)
log.info(modified_msg_count=len(modified))
new, updated = new_or_updated(modified, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
if new:
remote_g_metadata.update(crispin_client.g_metadata(new))
log.info('Updated cache with new messages')
else:
log.info('No new messages to update metadata for')
# Filter out messages that have disappeared.
old_len = len(remote_g_metadata)
current_remote_uids = set(crispin_client.all_uids())
remote_g_metadata = dict((uid, md)
for uid, md in remote_g_metadata.iteritems()
if uid in current_remote_uids)
num_removed = old_len - len(remote_g_metadata)
if num_removed > 0:
log.info(removed_msg_count=num_removed)
set_cache(
remote_g_metadata_cache_file(self.account_id, self.folder_name),
remote_g_metadata)
if updated:
# It's easy and fast to just update these here and now.
# Bigger chunk because the data being fetched here is very small.
for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE):
self.update_metadata(crispin_client, uids)
log.info('updated metadata for modified messages',
msg_count=len(updated))
return len(updated)
else:
log.info('No modified messages to update metadata for')
return 0
def check_flags(crispin_client, db_session, log, folder_name, local_uids,
syncmanager_lock):
""" Update message flags if folder has changed on the remote.
If we have saved validity info for this folder, make sure the folder hasn't
changed since we saved it. Otherwise we need to query for flag changes too.
"""
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
if saved_folder_info is not None:
last_highestmodseq = saved_folder_info.highestmodseq
if last_highestmodseq > crispin_client.selected_highestmodseq:
uids = crispin_client.new_and_updated_uids(last_highestmodseq)
if uids:
_, updated = new_or_updated(uids, local_uids)
update_metadata(crispin_client, db_session, log, folder_name,
updated, syncmanager_lock)
def check_flags(crispin_client, db_session, log, folder_name, local_uids,
syncmanager_lock):
""" Update message flags if folder has changed on the remote.
If we have saved validity info for this folder, make sure the folder hasn't
changed since we saved it. Otherwise we need to query for flag changes too.
"""
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
if saved_folder_info is not None:
last_highestmodseq = saved_folder_info.highestmodseq
if last_highestmodseq > crispin_client.selected_highestmodseq:
uids = crispin_client.new_and_updated_uids(last_highestmodseq)
if uids:
_, updated = new_or_updated(uids, local_uids)
update_metadata(crispin_client, db_session, log, folder_name,
updated, syncmanager_lock)
def highestmodseq_update(crispin_client, db_session, log, folder_name,
last_highestmodseq, highestmodseq_fn,
syncmanager_lock):
account_id = crispin_client.account_id
new_highestmodseq = crispin_client.selected_highestmodseq
new_uidvalidity = crispin_client.selected_uidvalidity
log.info("Starting highestmodseq update on {} (current HIGHESTMODSEQ: {})"
.format(folder_name, new_highestmodseq))
local_uids = account.all_uids(account_id, db_session, folder_name)
changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
remote_uids = crispin_client.all_uids()
if changed_uids:
new, updated = new_or_updated(changed_uids, local_uids)
log.info("{0} new and {1} updated UIDs".format(len(new), len(updated)))
local_uids += new
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
deleted_uids = remove_deleted_uids(account_id, db_session, log,
folder_name, local_uids,
remote_uids)
local_uids = set(local_uids) - deleted_uids
update_metadata(crispin_client, db_session, log, folder_name,
updated, syncmanager_lock)
update_uid_counts(db_session, log, account_id, folder_name,
remote_uid_count=len(remote_uids),
download_uid_count=len(new),
update_uid_count=len(updated),
delete_uid_count=len(deleted_uids))
highestmodseq_fn(crispin_client, db_session, log, folder_name,
changed_uids, local_uids, syncmanager_lock)
else:
log.info("No new or updated messages")
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
remove_deleted_uids(crispin_client.account_id, db_session, log,
folder_name, local_uids, remote_uids)
account.update_folder_info(account_id, db_session, folder_name,
new_uidvalidity, new_highestmodseq)
db_session.commit()
def highestmodseq_update(crispin_client, log, folder_name, last_highestmodseq,
highestmodseq_fn, syncmanager_lock):
account_id = crispin_client.account_id
new_highestmodseq = crispin_client.selected_highestmodseq
new_uidvalidity = crispin_client.selected_uidvalidity
log.info('starting highestmodseq update',
current_highestmodseq=new_highestmodseq)
changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
remote_uids = crispin_client.all_uids()
local_uids = None
if changed_uids:
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = account.all_uids(account_id, db_session, folder_name)
new, updated = new_or_updated(changed_uids, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
local_uids += new
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
deleted_uids = remove_deleted_uids(account_id, db_session, log,
folder_name, local_uids,
remote_uids)
local_uids = set(local_uids) - deleted_uids
update_metadata(crispin_client, log, folder_name, updated,
syncmanager_lock)
with session_scope(ignore_soft_deletes=False) as db_session:
update_uid_counts(db_session,
log,
account_id,
folder_name,
remote_uid_count=len(remote_uids),
download_uid_count=len(new),
update_uid_count=len(updated),
delete_uid_count=len(deleted_uids))
highestmodseq_fn(crispin_client, log, folder_name, new, updated,
syncmanager_lock)
else:
log.info("No new or updated messages")
with session_scope(ignore_soft_deletes=False) as db_session:
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
if local_uids is None:
local_uids = account.all_uids(account_id, db_session,
folder_name)
deleted_uids = remove_deleted_uids(crispin_client.account_id,
db_session, log, folder_name,
local_uids, remote_uids)
update_uid_counts(db_session,
log,
account_id,
folder_name,
remote_uid_count=len(remote_uids),
delete_uid_count=len(deleted_uids))
account.update_folder_info(account_id, db_session, folder_name,
new_uidvalidity, new_highestmodseq)
db_session.commit()