def get_g_metadata(crispin_client, db_session, log, folder_name, uids, syncmanager_lock): assert folder_name == crispin_client.selected_folder_name, \ "crispin selected folder isn't as expected" account_id = crispin_client.account_id remote_g_metadata = None saved_validity = account.get_uidvalidity(account_id, db_session, folder_name) if saved_validity is not None: # If there's no cached validity we probably haven't run before. remote_g_metadata = retrieve_saved_g_metadata(crispin_client, db_session, log, folder_name, uids, saved_validity, syncmanager_lock) if remote_g_metadata is None: remote_g_metadata = crispin_client.g_metadata( crispin_client.all_uids()) set_cache(remote_g_metadata_cache_file(account_id, folder_name), remote_g_metadata) # Save highestmodseq that corresponds to the saved g_metadata. account.update_uidvalidity(account_id, db_session, folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) db_session.commit() return remote_g_metadata
def __fetch_g_metadata(self, crispin_client, uids): assert self.folder_name == crispin_client.selected_folder_name, \ "crispin selected folder isn't as expected" remote_g_metadata = None with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info( self.account_id, db_session, self.folder_name) saved_highestmodseq = or_none(saved_folder_info, lambda i: i.highestmodseq) if saved_highestmodseq is not None: # If there's no cached validity we probably haven't run before. remote_g_metadata = self.__retrieve_saved_g_metadata( crispin_client, uids, saved_highestmodseq) if remote_g_metadata is None: remote_g_metadata = crispin_client.g_metadata( crispin_client.all_uids()) set_cache(remote_g_metadata_cache_file(self.account_id, self.folder_name), remote_g_metadata) # Save highestmodseq that corresponds to the saved g_metadata. with mailsync_session_scope() as db_session: common.update_folder_info(self.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) db_session.commit() return remote_g_metadata
def update_saved_g_metadata(crispin_client, db_session, log, folder_name, remote_g_metadata, local_uids, c): """ If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache, we need to query for any changes since then and update the saved data. """ log.info("Updating cache with latest changes") # any uids we don't already have will be downloaded correctly # as usual, but updated uids need to be updated manually # XXX it may actually be faster to just query for X-GM-MSGID for the # whole folder rather than getting changed UIDs first; MODSEQ queries # are slow on large folders. modified = crispin_client.new_and_updated_uids( crispin_client.selected_highestmodseq, c) new, updated = new_or_updated(modified, local_uids) log.info("{0} new and {1} updated UIDs".format(len(new), len(updated))) # for new, query metadata and update cache remote_g_metadata.update(crispin_client.g_metadata(new, c)) # filter out messages that have disappeared all_uids = set(crispin_client.all_uids(c)) remote_g_metadata = dict((uid, md) for uid, md in \ remote_g_metadata.iteritems() if uid in all_uids) set_cache(remote_g_metadata_cache_file(crispin_client.account_id, folder_name), remote_g_metadata) log.info("Updated cache with new messages") # for updated, it's easier to just update them now # bigger chunk because the data being fetched here is very small for uids in chunk(updated, 5*crispin_client.CHUNK_SIZE): update_metadata(crispin_client, db_session, log, folder_name, uids, c) log.info("Updated metadata for modified messages")
def __fetch_g_metadata(self, crispin_client, uids): assert self.folder_name == crispin_client.selected_folder_name, \ "crispin selected folder isn't as expected" remote_g_metadata = None update_uid_count = 0 with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info(self.account_id, db_session, self.folder_name) saved_highestmodseq = or_none(saved_folder_info, lambda i: i.highestmodseq) if saved_highestmodseq is not None: # If there's no cached validity we probably haven't run before. remote_g_metadata, update_uid_count = \ self.__retrieve_saved_g_metadata(crispin_client, uids, saved_highestmodseq) if remote_g_metadata is None: remote_g_metadata = crispin_client.g_metadata( crispin_client.all_uids()) set_cache( remote_g_metadata_cache_file(self.account_id, self.folder_name), remote_g_metadata) # Save highestmodseq that corresponds to the saved g_metadata. with mailsync_session_scope() as db_session: common.update_folder_info(self.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) db_session.commit() return remote_g_metadata, update_uid_count
def get_g_metadata(crispin_client, log, folder_name, uids, syncmanager_lock): assert folder_name == crispin_client.selected_folder_name, \ "crispin selected folder isn't as expected" account_id = crispin_client.account_id remote_g_metadata = None update_uid_count = 0 with session_scope(ignore_soft_deletes=False) as db_session: saved_folder_info = account.get_folder_info( account_id, db_session, folder_name) saved_highestmodseq = or_none(saved_folder_info, lambda i: i.highestmodseq) if saved_highestmodseq is not None: # If there's no cached validity we probably haven't run before. remote_g_metadata, update_uid_count = retrieve_saved_g_metadata( crispin_client, log, folder_name, uids, saved_highestmodseq, syncmanager_lock) if remote_g_metadata is None: remote_g_metadata = crispin_client.g_metadata( crispin_client.all_uids()) set_cache(remote_g_metadata_cache_file(account_id, folder_name), remote_g_metadata) # Save highestmodseq that corresponds to the saved g_metadata. with session_scope(ignore_soft_deletes=False) as db_session: account.update_folder_info(account_id, db_session, folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) db_session.commit() return remote_g_metadata, update_uid_count
def __update_saved_g_metadata(self, crispin_client, remote_g_metadata, local_uids): """ If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache, we need to query for any changes since then and update the saved data. """ log.info('Updating cache with latest changes') # Any uids we don't already have will be downloaded correctly as usual, # but updated uids need to be updated manually. # XXX it may actually be faster to just query for X-GM-MSGID for the # whole folder rather than getting changed UIDs first; MODSEQ queries # are slow on large folders. modified = crispin_client.new_and_updated_uids( crispin_client.selected_highestmodseq) log.info(modified_msg_count=len(modified)) new, updated = new_or_updated(modified, local_uids) log.info(new_uid_count=len(new), updated_uid_count=len(updated)) if new: remote_g_metadata.update(crispin_client.g_metadata(new)) log.info('Updated cache with new messages') else: log.info('No new messages to update metadata for') # Filter out messages that have disappeared. old_len = len(remote_g_metadata) current_remote_uids = set(crispin_client.all_uids()) remote_g_metadata = dict((uid, md) for uid, md in remote_g_metadata.iteritems() if uid in current_remote_uids) num_removed = old_len - len(remote_g_metadata) if num_removed > 0: log.info(removed_msg_count=num_removed) set_cache(remote_g_metadata_cache_file(self.account_id, self.folder_name), remote_g_metadata) if updated: # It's easy and fast to just update these here and now. # Bigger chunk because the data being fetched here is very small. for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE): self.update_metadata(crispin_client, uids) log.info('updated metadata for modified messages', msg_count=len(updated)) return len(updated) else: log.info('No modified messages to update metadata for') return 0
def __update_saved_g_metadata(self, crispin_client, remote_g_metadata, local_uids): """ If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache, we need to query for any changes since then and update the saved data. """ log.info('Updating cache with latest changes') # Any uids we don't already have will be downloaded correctly as usual, # but updated uids need to be updated manually. # XXX it may actually be faster to just query for X-GM-MSGID for the # whole folder rather than getting changed UIDs first; MODSEQ queries # are slow on large folders. modified = crispin_client.new_and_updated_uids( crispin_client.selected_highestmodseq) log.info(modified_msg_count=len(modified)) new, updated = new_or_updated(modified, local_uids) log.info(new_uid_count=len(new), updated_uid_count=len(updated)) if new: remote_g_metadata.update(crispin_client.g_metadata(new)) log.info('Updated cache with new messages') else: log.info('No new messages to update metadata for') # Filter out messages that have disappeared. old_len = len(remote_g_metadata) current_remote_uids = set(crispin_client.all_uids()) remote_g_metadata = dict((uid, md) for uid, md in remote_g_metadata.iteritems() if uid in current_remote_uids) num_removed = old_len - len(remote_g_metadata) if num_removed > 0: log.info(removed_msg_count=num_removed) set_cache( remote_g_metadata_cache_file(self.account_id, self.folder_name), remote_g_metadata) if updated: # It's easy and fast to just update these here and now. # Bigger chunk because the data being fetched here is very small. for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE): self.update_metadata(crispin_client, uids) log.info('updated metadata for modified messages', msg_count=len(updated)) return len(updated) else: log.info('No modified messages to update metadata for') return 0