def get_new_uids(self, crispin_client): try: remote_uidnext = crispin_client.conn.folder_status( self.folder_name, ['UIDNEXT']).get('UIDNEXT') except ValueError: # Work around issue where ValueError is raised on parsing STATUS # response. log.warning('Error getting UIDNEXT', exc_info=True) remote_uidnext = None except imaplib.IMAP4.error as e: if '[NONEXISTENT]' in e.message: raise FolderMissingError() else: raise e if remote_uidnext is not None and remote_uidnext == self.uidnext: return log.info('UIDNEXT changed, checking for new UIDs', remote_uidnext=remote_uidnext, saved_uidnext=self.uidnext) crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) with session_scope(self.namespace_id) as db_session: lastseenuid = common.lastseenuid(self.account_id, db_session, self.folder_id) latest_uids = crispin_client.conn.fetch('{}:*'.format(lastseenuid + 1), ['UID']).keys() new_uids = set(latest_uids) - {lastseenuid} if new_uids: for uid in sorted(new_uids): self.download_and_commit_uids(crispin_client, [uid]) self.uidnext = remote_uidnext
def condstore_refresh_flags(self, crispin_client): new_highestmodseq = crispin_client.conn.folder_status( self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ'] # Ensure that we have an initial highestmodseq value stored before we # begin polling for changes. if self.highestmodseq is None: self.highestmodseq = new_highestmodseq if new_highestmodseq == self.highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < self.highestmodseq: # This should really never happen, but if it does, handle it. log.warning( 'got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=self.highestmodseq) return # Highestmodseq has changed, update accordingly. crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) changed_flags = crispin_client.condstore_changed_flags( self.highestmodseq) remote_uids = crispin_client.all_uids() with session_scope(self.namespace_id) as db_session: common.update_metadata(self.account_id, self.folder_id, changed_flags, db_session) local_uids = common.local_uids(self.account_id, db_session, self.folder_id) expunged_uids = set(local_uids).difference(remote_uids) if expunged_uids: # If new UIDs have appeared since we last checked in # get_new_uids, save them first. We want to always have the # latest UIDs before expunging anything, in order to properly # capture draft revisions. with session_scope(self.namespace_id) as db_session: lastseenuid = common.lastseenuid(self.account_id, db_session, self.folder_id) if remote_uids and lastseenuid < max(remote_uids): log.info('Downloading new UIDs before expunging') self.get_new_uids(crispin_client) with session_scope(self.namespace_id) as db_session: common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids, db_session) db_session.commit() self.highestmodseq = new_highestmodseq
def condstore_refresh_flags(self, crispin_client): new_highestmodseq = crispin_client.conn.folder_status( self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ'] # Ensure that we have an initial highestmodseq value stored before we # begin polling for changes. if self.highestmodseq is None: self.highestmodseq = new_highestmodseq if new_highestmodseq == self.highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < self.highestmodseq: # This should really never happen, but if it does, handle it. log.warning('got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=self.highestmodseq) return # Highestmodseq has changed, update accordingly. crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) changed_flags = crispin_client.condstore_changed_flags( self.highestmodseq) remote_uids = crispin_client.all_uids() with session_scope(self.namespace_id) as db_session: common.update_metadata(self.account_id, self.folder_id, changed_flags, db_session) local_uids = common.local_uids(self.account_id, db_session, self.folder_id) expunged_uids = set(local_uids).difference(remote_uids) if expunged_uids: # If new UIDs have appeared since we last checked in # get_new_uids, save them first. We want to always have the # latest UIDs before expunging anything, in order to properly # capture draft revisions. with session_scope(self.namespace_id) as db_session: lastseenuid = common.lastseenuid(self.account_id, db_session, self.folder_id) if remote_uids and lastseenuid < max(remote_uids): log.info('Downloading new UIDs before expunging') self.get_new_uids(crispin_client) with session_scope(self.namespace_id) as db_session: common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids, db_session) db_session.commit() self.highestmodseq = new_highestmodseq
def get_new_uids(self, crispin_client): try: remote_uidnext = crispin_client.conn.folder_status( self.folder_name, ["UIDNEXT"]).get(b"UIDNEXT") except ValueError: # Work around issue where ValueError is raised on parsing STATUS # response. log.warning("Error getting UIDNEXT", exc_info=True) remote_uidnext = None except imaplib.IMAP4.error as e: # TODO: match with CrispinClient.select_folder message = e.args[0] if e.args else "" if ("[NONEXISTENT]" in message or "does not exist" in message or "doesn't exist" in message): raise FolderMissingError() else: raise e if remote_uidnext is not None and remote_uidnext == self.uidnext: return log.debug( "UIDNEXT changed, checking for new UIDs", remote_uidnext=remote_uidnext, saved_uidnext=self.uidnext, ) crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) with session_scope(self.namespace_id) as db_session: lastseenuid = common.lastseenuid(self.account_id, db_session, self.folder_id) latest_uids = crispin_client.conn.fetch("{}:*".format(lastseenuid + 1), ["UID"]).keys() new_uids = set(latest_uids) - {lastseenuid} if new_uids: for uid in sorted(new_uids): self.download_and_commit_uids(crispin_client, [uid]) self.uidnext = remote_uidnext
def condstore_refresh_flags(self, crispin_client): new_highestmodseq = crispin_client.conn.folder_status( self.folder_name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ'] # Ensure that we have an initial highestmodseq value stored before we # begin polling for changes. if self.highestmodseq is None: self.highestmodseq = new_highestmodseq if new_highestmodseq == self.highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < self.highestmodseq: # This should really never happen, but if it does, handle it. log.warning('got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=self.highestmodseq) return log.info('HIGHESTMODSEQ has changed, getting changed UIDs', new_highestmodseq=new_highestmodseq, saved_highestmodseq=self.highestmodseq) crispin_client.select_folder(self.folder_name, self.uidvalidity_cb) changed_flags = crispin_client.condstore_changed_flags( self.highestmodseq) remote_uids = crispin_client.all_uids() # In order to be able to sync changes to tens of thousands of flags at # once, we commit updates in batches. We do this in ascending order by # modseq and periodically "checkpoint" our saved highestmodseq. (It's # safe to checkpoint *because* we go in ascending order by modseq.) # That way if the process gets restarted halfway through this refresh, # we don't have to completely start over. It's also slow to load many # objects into the SQLAlchemy session and then issue lots of commits; # we avoid that by batching. flag_batches = chunk( sorted(changed_flags.items(), key=lambda (k, v): v.modseq), CONDSTORE_FLAGS_REFRESH_BATCH_SIZE) for flag_batch in flag_batches: with session_scope(self.namespace_id) as db_session: common.update_metadata(self.account_id, self.folder_id, self.folder_role, dict(flag_batch), db_session) if len(flag_batch) == CONDSTORE_FLAGS_REFRESH_BATCH_SIZE: interim_highestmodseq = max(v.modseq for k, v in flag_batch) self.highestmodseq = interim_highestmodseq with session_scope(self.namespace_id) as db_session: local_uids = common.local_uids(self.account_id, db_session, self.folder_id) expunged_uids = set(local_uids).difference(remote_uids) if expunged_uids: # If new UIDs have appeared since we last checked in # get_new_uids, save them first. We want to always have the # latest UIDs before expunging anything, in order to properly # capture draft revisions. with session_scope(self.namespace_id) as db_session: lastseenuid = common.lastseenuid(self.account_id, db_session, self.folder_id) if remote_uids and lastseenuid < max(remote_uids): log.info('Downloading new UIDs before expunging') self.get_new_uids(crispin_client) common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids) self.highestmodseq = new_highestmodseq