def get_g_metadata(crispin_client, db_session, log, folder_name, uids,
syncmanager_lock):
assert folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
account_id = crispin_client.account_id
remote_g_metadata = None
saved_validity = account.get_uidvalidity(account_id, db_session,
folder_name)
if saved_validity is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata = retrieve_saved_g_metadata(crispin_client,
db_session, log,
folder_name, uids,
saved_validity,
syncmanager_lock)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(remote_g_metadata_cache_file(account_id, folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
account.update_uidvalidity(account_id, db_session, folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata
def __fetch_g_metadata(self, crispin_client, uids):
assert self.folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
remote_g_metadata = None
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(
self.account_id, db_session, self.folder_name)
saved_highestmodseq = or_none(saved_folder_info, lambda i:
i.highestmodseq)
if saved_highestmodseq is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata = self.__retrieve_saved_g_metadata(
crispin_client, uids, saved_highestmodseq)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(remote_g_metadata_cache_file(self.account_id,
self.folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
with mailsync_session_scope() as db_session:
common.update_folder_info(self.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata
def update_saved_g_metadata(crispin_client, db_session, log, folder_name,
remote_g_metadata, local_uids, c):
""" If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache,
we need to query for any changes since then and update the saved
data.
"""
log.info("Updating cache with latest changes")
# any uids we don't already have will be downloaded correctly
# as usual, but updated uids need to be updated manually
# XXX it may actually be faster to just query for X-GM-MSGID for the
# whole folder rather than getting changed UIDs first; MODSEQ queries
# are slow on large folders.
modified = crispin_client.new_and_updated_uids(
crispin_client.selected_highestmodseq, c)
new, updated = new_or_updated(modified, local_uids)
log.info("{0} new and {1} updated UIDs".format(len(new), len(updated)))
# for new, query metadata and update cache
remote_g_metadata.update(crispin_client.g_metadata(new, c))
# filter out messages that have disappeared
all_uids = set(crispin_client.all_uids(c))
remote_g_metadata = dict((uid, md) for uid, md in \
remote_g_metadata.iteritems() if uid in all_uids)
set_cache(remote_g_metadata_cache_file(crispin_client.account_id,
folder_name), remote_g_metadata)
log.info("Updated cache with new messages")
# for updated, it's easier to just update them now
# bigger chunk because the data being fetched here is very small
for uids in chunk(updated, 5*crispin_client.CHUNK_SIZE):
update_metadata(crispin_client, db_session, log, folder_name, uids, c)
log.info("Updated metadata for modified messages")
def __fetch_g_metadata(self, crispin_client, uids):
assert self.folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
remote_g_metadata = None
update_uid_count = 0
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
saved_highestmodseq = or_none(saved_folder_info,
lambda i: i.highestmodseq)
if saved_highestmodseq is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata, update_uid_count = \
self.__retrieve_saved_g_metadata(crispin_client, uids,
saved_highestmodseq)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(
remote_g_metadata_cache_file(self.account_id,
self.folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
with mailsync_session_scope() as db_session:
common.update_folder_info(self.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata, update_uid_count
def get_g_metadata(crispin_client, log, folder_name, uids, syncmanager_lock):
assert folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
account_id = crispin_client.account_id
remote_g_metadata = None
update_uid_count = 0
with session_scope(ignore_soft_deletes=False) as db_session:
saved_folder_info = account.get_folder_info(
account_id, db_session, folder_name)
saved_highestmodseq = or_none(saved_folder_info, lambda i:
i.highestmodseq)
if saved_highestmodseq is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata, update_uid_count = retrieve_saved_g_metadata(
crispin_client, log, folder_name, uids,
saved_highestmodseq, syncmanager_lock)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(remote_g_metadata_cache_file(account_id, folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
with session_scope(ignore_soft_deletes=False) as db_session:
account.update_folder_info(account_id, db_session, folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata, update_uid_count
def get_g_metadata(crispin_client, db_session, log, folder_name, uids,
syncmanager_lock):
assert folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
account_id = crispin_client.account_id
remote_g_metadata = None
saved_validity = account.get_uidvalidity(account_id, db_session,
folder_name)
if saved_validity is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata = retrieve_saved_g_metadata(crispin_client,
db_session, log,
folder_name, uids,
saved_validity,
syncmanager_lock)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(remote_g_metadata_cache_file(account_id, folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
account.update_uidvalidity(account_id, db_session, folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata
def __update_saved_g_metadata(self, crispin_client, remote_g_metadata,
local_uids):
"""
If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache,
we need to query for any changes since then and update the saved
data.
"""
log.info('Updating cache with latest changes')
# Any uids we don't already have will be downloaded correctly as usual,
# but updated uids need to be updated manually.
# XXX it may actually be faster to just query for X-GM-MSGID for the
# whole folder rather than getting changed UIDs first; MODSEQ queries
# are slow on large folders.
modified = crispin_client.new_and_updated_uids(
crispin_client.selected_highestmodseq)
log.info(modified_msg_count=len(modified))
new, updated = new_or_updated(modified, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
if new:
remote_g_metadata.update(crispin_client.g_metadata(new))
log.info('Updated cache with new messages')
else:
log.info('No new messages to update metadata for')
# Filter out messages that have disappeared.
old_len = len(remote_g_metadata)
current_remote_uids = set(crispin_client.all_uids())
remote_g_metadata = dict((uid, md) for uid, md in
remote_g_metadata.iteritems() if uid in
current_remote_uids)
num_removed = old_len - len(remote_g_metadata)
if num_removed > 0:
log.info(removed_msg_count=num_removed)
set_cache(remote_g_metadata_cache_file(self.account_id,
self.folder_name),
remote_g_metadata)
if updated:
# It's easy and fast to just update these here and now.
# Bigger chunk because the data being fetched here is very small.
for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE):
self.update_metadata(crispin_client, uids)
log.info('updated metadata for modified messages',
msg_count=len(updated))
return len(updated)
else:
log.info('No modified messages to update metadata for')
return 0
def __update_saved_g_metadata(self, crispin_client, remote_g_metadata,
local_uids):
"""
If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache,
we need to query for any changes since then and update the saved
data.
"""
log.info('Updating cache with latest changes')
# Any uids we don't already have will be downloaded correctly as usual,
# but updated uids need to be updated manually.
# XXX it may actually be faster to just query for X-GM-MSGID for the
# whole folder rather than getting changed UIDs first; MODSEQ queries
# are slow on large folders.
modified = crispin_client.new_and_updated_uids(
crispin_client.selected_highestmodseq)
log.info(modified_msg_count=len(modified))
new, updated = new_or_updated(modified, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
if new:
remote_g_metadata.update(crispin_client.g_metadata(new))
log.info('Updated cache with new messages')
else:
log.info('No new messages to update metadata for')
# Filter out messages that have disappeared.
old_len = len(remote_g_metadata)
current_remote_uids = set(crispin_client.all_uids())
remote_g_metadata = dict((uid, md)
for uid, md in remote_g_metadata.iteritems()
if uid in current_remote_uids)
num_removed = old_len - len(remote_g_metadata)
if num_removed > 0:
log.info(removed_msg_count=num_removed)
set_cache(
remote_g_metadata_cache_file(self.account_id, self.folder_name),
remote_g_metadata)
if updated:
# It's easy and fast to just update these here and now.
# Bigger chunk because the data being fetched here is very small.
for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE):
self.update_metadata(crispin_client, uids)
log.info('updated metadata for modified messages',
msg_count=len(updated))
return len(updated)
else:
log.info('No modified messages to update metadata for')
return 0
