def check_uid_changes(self, crispin_client, download_stack, async_download): crispin_client.select_folder(self.folder_name, uidvalidity_cb) new_highestmodseq = crispin_client.selected_highestmodseq with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info(self.account_id, db_session, self.folder_name) # Ensure that we have an initial highestmodseq value stored before # we begin polling for changes. if saved_folder_info is None or \ saved_folder_info.highestmodseq is None: assert (crispin_client.selected_uidvalidity is not None and crispin_client.selected_highestmodseq is not None) saved_folder_info = common.update_folder_info( crispin_client.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) saved_highestmodseq = saved_folder_info.highestmodseq if new_highestmodseq == saved_highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < saved_highestmodseq: # This should really never happen, but if it does, handle it. log.warning( 'got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=saved_highestmodseq) return # Highestmodseq has changed, update accordingly. new_uidvalidity = crispin_client.selected_uidvalidity changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq) remote_uids = crispin_client.all_uids() with mailsync_session_scope() as db_session: local_uids = common.all_uids(self.account_id, db_session, self.folder_id) stack_uids = set(download_stack.keys()) local_with_pending_uids = local_uids | stack_uids new, updated = new_or_updated(changed_uids, local_with_pending_uids) if changed_uids: log.info("Changed UIDs", message="new: {} updated: {}".format( len(new), len(updated)), new_uid_count=len(new), updated_uid_count=len(updated)) self.update_metadata(crispin_client, updated) self.highestmodseq_callback(crispin_client, new, updated, download_stack, async_download) with mailsync_session_scope() as db_session: with self.syncmanager_lock: self.remove_deleted_uids(db_session, local_uids, remote_uids) self.update_uid_counts(db_session, remote_uid_count=len(remote_uids)) common.update_folder_info(self.account_id, db_session, self.folder_name, new_uidvalidity, new_highestmodseq) db_session.commit()
def __fetch_g_metadata(self, crispin_client, uids): assert self.folder_name == crispin_client.selected_folder_name, \ "crispin selected folder isn't as expected" remote_g_metadata = None with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info( self.account_id, db_session, self.folder_name) saved_highestmodseq = or_none(saved_folder_info, lambda i: i.highestmodseq) if saved_highestmodseq is not None: # If there's no cached validity we probably haven't run before. remote_g_metadata = self.__retrieve_saved_g_metadata( crispin_client, uids, saved_highestmodseq) if remote_g_metadata is None: remote_g_metadata = crispin_client.g_metadata( crispin_client.all_uids()) set_cache(remote_g_metadata_cache_file(self.account_id, self.folder_name), remote_g_metadata) # Save highestmodseq that corresponds to the saved g_metadata. with mailsync_session_scope() as db_session: common.update_folder_info(self.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) db_session.commit() return remote_g_metadata
def check_uid_changes(self, crispin_client, download_stack, async_download): crispin_client.select_folder(self.folder_name, uidvalidity_cb) new_highestmodseq = crispin_client.selected_highestmodseq with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info( self.account_id, db_session, self.folder_name) # Ensure that we have an initial highestmodseq value stored before # we begin polling for changes. if saved_folder_info is None or \ saved_folder_info.highestmodseq is None: assert (crispin_client.selected_uidvalidity is not None and crispin_client.selected_highestmodseq is not None) saved_folder_info = common.update_folder_info( crispin_client.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) saved_highestmodseq = saved_folder_info.highestmodseq if new_highestmodseq == saved_highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < saved_highestmodseq: # This should really never happen, but if it does, handle it. log.warning('got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=saved_highestmodseq) return save_folder_names(log, self.account_id, crispin_client.folder_names(), db_session) # Highestmodseq has changed, update accordingly. new_uidvalidity = crispin_client.selected_uidvalidity changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq) remote_uids = crispin_client.all_uids() with mailsync_session_scope() as db_session: local_uids = common.all_uids(self.account_id, db_session, self.folder_name) stack_uids = {uid for uid, _ in download_stack} local_with_pending_uids = local_uids | stack_uids new, updated = new_or_updated(changed_uids, local_with_pending_uids) if changed_uids: log.info("Changed UIDs", message="new: {} updated: {}" .format(len(new), len(updated)), new_uid_count=len(new), updated_uid_count=len(updated)) self.update_metadata(crispin_client, updated) self.highestmodseq_callback(crispin_client, new, updated, download_stack, async_download) with mailsync_session_scope() as db_session: with self.syncmanager_lock: self.remove_deleted_uids(db_session, local_uids, remote_uids) self.update_uid_counts(db_session, remote_uid_count=len(remote_uids)) common.update_folder_info(self.account_id, db_session, self.folder_name, new_uidvalidity, new_highestmodseq) db_session.commit()
def __fetch_g_metadata(self, crispin_client, uids): assert self.folder_name == crispin_client.selected_folder_name, \ "crispin selected folder isn't as expected" remote_g_metadata = None update_uid_count = 0 with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info(self.account_id, db_session, self.folder_name) saved_highestmodseq = or_none(saved_folder_info, lambda i: i.highestmodseq) if saved_highestmodseq is not None: # If there's no cached validity we probably haven't run before. remote_g_metadata, update_uid_count = \ self.__retrieve_saved_g_metadata(crispin_client, uids, saved_highestmodseq) if remote_g_metadata is None: remote_g_metadata = crispin_client.g_metadata( crispin_client.all_uids()) set_cache( remote_g_metadata_cache_file(self.account_id, self.folder_name), remote_g_metadata) # Save highestmodseq that corresponds to the saved g_metadata. with mailsync_session_scope() as db_session: common.update_folder_info(self.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) db_session.commit() return remote_g_metadata, update_uid_count
def check_uid_changes(self, crispin_client, download_stack, async_download): remote_uids = set(crispin_client.all_uids()) with self.syncmanager_lock: with mailsync_session_scope() as db_session: local_uids = common.all_uids(self.account_id, db_session, self.folder_id) # Download new UIDs. stack_uids = set(download_stack.keys()) local_with_pending_uids = local_uids | stack_uids for uid in sorted(remote_uids): if uid not in local_with_pending_uids: download_stack.put(uid, None) self.remove_deleted_uids(db_session, local_uids, remote_uids) if not async_download: self.download_uids(crispin_client, download_stack) with mailsync_session_scope() as db_session: self.update_uid_counts( db_session, remote_uid_count=len(remote_uids), download_uid_count=len(download_stack)) to_refresh = sorted(remote_uids & local_uids)[-self.refresh_flags_max:] self.update_metadata(crispin_client, to_refresh) with mailsync_session_scope() as db_session: common.update_folder_info(self.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, None, crispin_client.selected_uidnext)
def highestmodseq_update(self, crispin_client, last_highestmodseq): new_highestmodseq = crispin_client.selected_highestmodseq new_uidvalidity = crispin_client.selected_uidvalidity log.info('starting highestmodseq update', current_highestmodseq=new_highestmodseq) changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq) remote_uids = crispin_client.all_uids() local_uids = None if changed_uids: with mailsync_session_scope() as db_session: local_uids = common.all_uids(self.account_id, db_session, self.folder_name) new, updated = new_or_updated(changed_uids, local_uids) log.info(new_uid_count=len(new), updated_uid_count=len(updated)) local_uids.update(new) with self.syncmanager_lock: log.debug("highestmodseq_update acquired syncmanager_lock") with mailsync_session_scope() as db_session: deleted_uids = self.remove_deleted_uids( db_session, local_uids, remote_uids) local_uids = local_uids - deleted_uids self.update_metadata(crispin_client, updated) with mailsync_session_scope() as db_session: self.update_uid_counts( db_session, remote_uid_count=len(remote_uids), download_uid_count=len(new), update_uid_count=len(updated), delete_uid_count=len(deleted_uids)) self.highestmodseq_callback(crispin_client, new, updated) else: log.info("No new or updated messages") with mailsync_session_scope() as db_session: with self.syncmanager_lock: log.debug("highestmodseq_update acquired syncmanager_lock") if local_uids is None: local_uids = common.all_uids( self.account_id, db_session, self.folder_name) deleted_uids = self.remove_deleted_uids( db_session, local_uids, remote_uids) self.update_uid_counts(db_session, remote_uid_count=len(remote_uids), delete_uid_count=len(deleted_uids)) common.update_folder_info(self.account_id, db_session, self.folder_name, new_uidvalidity, new_highestmodseq) db_session.commit()
def initial_sync_impl(self, crispin_client, local_uids, uid_download_stack): with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info(self.account_id, db_session, self.folder_name) if saved_folder_info is None: assert (crispin_client.selected_uidvalidity is not None and crispin_client.selected_highestmodseq is not None) common.update_folder_info( crispin_client.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) self.__check_flags(crispin_client, db_session, local_uids) return FolderSyncEngine.initial_sync_impl( self, crispin_client, local_uids, uid_download_stack, spawn_flags_refresh_poller=False)