def test_handle_trailing_whitespace(db, default_account, folder_name_mapping): folder_name_mapping["extra"] = ["label", "label "] log = get_logger() save_folder_names(log, default_account.id, folder_name_mapping, db.session) # Would raise if tag for label was not committed. db.session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="label").one()
def test_sync_folder_deletes(db, default_account, folder_name_mapping): """Test that folder deletions properly cascade to deletions of ImapFolderSyncStatus and ImapFolderInfo. """ with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, default_account.id, folder_name_mapping, db_session) folders = db_session.query(Folder).filter_by(account_id=default_account.id) for folder in folders: add_imap_status_info_rows(folder.id, default_account.id, db_session) db_session.commit() assert db_session.query(ImapFolderInfo).filter_by(account_id=default_account.id).count() == 7 assert db_session.query(ImapFolderSyncStatus).filter_by(account_id=default_account.id).count() == 7 folder_name_mapping["extra"] = ["Jobslist"] save_folder_names(log, default_account.id, folder_name_mapping, db_session) saved_folder_names = { name for name, in db_session.query(Folder.name).filter(Folder.account_id == default_account.id) } assert saved_folder_names == { "Inbox", "[Gmail]/Spam", "[Gmail]/All Mail", "[Gmail]/Sent Mail", "[Gmail]/Drafts", "Jobslist", } assert db_session.query(ImapFolderInfo).filter_by(account_id=default_account.id).count() == 6 assert db_session.query(ImapFolderSyncStatus).filter_by(account_id=default_account.id).count() == 6
def prepare_sync(self): """Ensures that canonical tags are created for the account, and gets and save Folder objects for folders on the IMAP backend. Returns a list of tuples (folder_name, folder_id) for each folder we want to sync (in order).""" with mailsync_session_scope() as db_session: with connection_pool(self.account_id).get() as crispin_client: # the folders we should be syncing sync_folders = crispin_client.sync_folders() # get a fresh list of the folder names from the remote remote_folders = crispin_client.folder_names(force_resync=True) save_folder_names(log, self.account_id, remote_folders, db_session) sync_folder_names_ids = [] for folder_name in sync_folders: try: id_, = db_session.query(Folder.id). \ filter(Folder.name == folder_name, Folder.account_id == self.account_id).one() sync_folder_names_ids.append((folder_name, id_)) except NoResultFound: log.error("Missing Folder object when starting sync", folder_name=folder_name) raise MailsyncError("Missing Folder '{}' on account {}" .format(folder_name, self.account_id)) return sync_folder_names_ids
def check_uid_changes(self, crispin_client, download_stack, async_download): crispin_client.select_folder(self.folder_name, uidvalidity_cb) new_highestmodseq = crispin_client.selected_highestmodseq with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info( self.account_id, db_session, self.folder_name) # Ensure that we have an initial highestmodseq value stored before # we begin polling for changes. if saved_folder_info is None or \ saved_folder_info.highestmodseq is None: assert (crispin_client.selected_uidvalidity is not None and crispin_client.selected_highestmodseq is not None) saved_folder_info = common.update_folder_info( crispin_client.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) saved_highestmodseq = saved_folder_info.highestmodseq if new_highestmodseq == saved_highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < saved_highestmodseq: # This should really never happen, but if it does, handle it. log.warning('got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=saved_highestmodseq) return save_folder_names(log, self.account_id, crispin_client.folder_names(), db_session) # Highestmodseq has changed, update accordingly. new_uidvalidity = crispin_client.selected_uidvalidity changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq) remote_uids = crispin_client.all_uids() with mailsync_session_scope() as db_session: local_uids = common.all_uids(self.account_id, db_session, self.folder_name) stack_uids = {uid for uid, _ in download_stack} local_with_pending_uids = local_uids | stack_uids new, updated = new_or_updated(changed_uids, local_with_pending_uids) if changed_uids: log.info("Changed UIDs", message="new: {} updated: {}" .format(len(new), len(updated)), new_uid_count=len(new), updated_uid_count=len(updated)) self.update_metadata(crispin_client, updated) self.highestmodseq_callback(crispin_client, new, updated, download_stack, async_download) with mailsync_session_scope() as db_session: with self.syncmanager_lock: self.remove_deleted_uids(db_session, local_uids, remote_uids) self.update_uid_counts(db_session, remote_uid_count=len(remote_uids)) common.update_folder_info(self.account_id, db_session, self.folder_name, new_uidvalidity, new_highestmodseq) db_session.commit()
def prepare_sync(self): """Ensures that canonical tags are created for the account, and gets and save Folder objects for folders on the IMAP backend. Returns a list of tuples (folder_name, folder_id) for each folder we want to sync (in order).""" with mailsync_session_scope() as db_session: account = db_session.query(ImapAccount).get(self.account_id) Tag.create_canonical_tags(account.namespace, db_session) with _pool(self.account_id).get() as crispin_client: sync_folders = crispin_client.sync_folders() save_folder_names(log, self.account_id, crispin_client.folder_names(), db_session) sync_folder_names_ids = [] for folder_name in sync_folders: try: id_, = db_session.query(Folder.id). \ filter(Folder.name == folder_name, Folder.account_id == self.account_id).one() sync_folder_names_ids.append((folder_name, id_)) except NoResultFound: log.error("Missing Folder object when starting sync", folder_name=folder_name) raise MailsyncError("Missing Folder '{}' on account {}" .format(folder_name, self.account_id)) return sync_folder_names_ids
def test_name_collision_folders(db, default_account, folder_name_mapping): # test that when a user-created folder called 'spam' is created, we don't # associate it with the canonical spam tag, but instead give it its own # tag folder_name_mapping["extra"] = ["spam"] with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, default_account.id, folder_name_mapping, db_session) spam_tags = db_session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="spam") # There should be one 'Gmail/Spam' canonical tag assert spam_tags.count() == 1 assert spam_tags.first().public_id == "spam" # and one 'imap/spam' non-canonical tag with public_id != 'spam' spam_tags = db_session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="imap/spam") assert spam_tags.count() == 1 assert spam_tags.first().public_id != "spam" # test that when a folder called 'spam' is deleted, we don't delete # the canonical 'spam' tag folder_name_mapping["extra"] = [] with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, default_account.id, folder_name_mapping, db_session) spam_tags = db_session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="spam") # The 'Gmail/Spam' canonical tag should still remain. assert spam_tags.count() == 1 assert spam_tags.first().public_id == "spam" # The 'imap/spam' non-canonical tag shouldn't spam_tags = db_session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="imap/spam") assert spam_tags.count() == 0
def test_handle_trailing_whitespace(db, default_account, folder_name_mapping): folder_name_mapping['extra'] = ['label', 'label '] log = get_logger() save_folder_names(log, default_account.id, folder_name_mapping, db.session) # Would raise if tag for label was not committed. db.session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name='label').one()
def create_foldersyncstatuses(db, default_account): # Create a bunch of folder sync statuses. mapping = folder_name_mapping() save_folder_names(log, default_account.id, mapping, db.session) folders = db.session.query(Folder).filter_by(account_id=default_account.id) for folder in folders: add_imap_status_info_rows(folder.id, default_account.id, db.session) db.session.commit()
def check_uid_changes(self, crispin_client, download_stack, async_download): crispin_client.select_folder(self.folder_name, uidvalidity_cb) new_highestmodseq = crispin_client.selected_highestmodseq with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info(self.account_id, db_session, self.folder_name) # Ensure that we have an initial highestmodseq value stored before # we begin polling for changes. if saved_folder_info is None: assert (crispin_client.selected_uidvalidity is not None and crispin_client.selected_highestmodseq is not None) saved_folder_info = common.update_folder_info( crispin_client.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) saved_highestmodseq = saved_folder_info.highestmodseq if new_highestmodseq == saved_highestmodseq: # Don't need to do anything if the highestmodseq hasn't # changed. return elif new_highestmodseq < saved_highestmodseq: # This should really never happen, but if it does, handle it. log.warning( 'got server highestmodseq less than saved ' 'highestmodseq', new_highestmodseq=new_highestmodseq, saved_highestmodseq=saved_highestmodseq) return save_folder_names(log, self.account_id, crispin_client.folder_names(), db_session) # Highestmodseq has changed, update accordingly. new_uidvalidity = crispin_client.selected_uidvalidity changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq) remote_uids = crispin_client.all_uids() with self.syncmanager_lock: with mailsync_session_scope() as db_session: local_uids = common.all_uids(self.account_id, db_session, self.folder_name) self.remove_deleted_uids(db_session, local_uids, remote_uids) stack_uids = {uid for uid, _ in download_stack} local_with_pending_uids = local_uids | stack_uids new, updated = new_or_updated(changed_uids, local_with_pending_uids) if changed_uids: log.info(new_uid_count=len(new), updated_uid_count=len(updated)) self.update_metadata(crispin_client, updated) self.highestmodseq_callback(crispin_client, new, updated, download_stack, async_download) with mailsync_session_scope() as db_session: self.update_uid_counts(db_session, remote_uid_count=len(remote_uids)) common.update_folder_info(self.account_id, db_session, self.folder_name, new_uidvalidity, new_highestmodseq) db_session.commit()
def sync(self): """ Start per-folder syncs. Only have one per-folder sync in the 'initial' state at a time. """ with mailsync_session_scope() as db_session: with _pool(self.account_id).get() as crispin_client: sync_folders = crispin_client.sync_folders() account = db_session.query(ImapAccount)\ .get(self.account_id) save_folder_names(log, account, crispin_client.folder_names(), db_session) Tag.create_canonical_tags(account.namespace, db_session) folder_id_for = { name: id_ for id_, name in db_session.query(Folder.id, Folder.name). filter_by(account_id=self.account_id) } saved_states = { name: state for name, state in db_session.query( Folder.name, ImapFolderSyncStatus.state).join( ImapFolderSyncStatus.folder).filter( ImapFolderSyncStatus.account_id == self.account_id) } for folder_name in sync_folders: if folder_name not in folder_id_for: log.error("Missing Folder object when starting sync", folder_name=folder_name, folder_id_for=folder_id_for) raise MailsyncError("Missing Folder '{}' on account {}".format( folder_name, self.account_id)) if saved_states.get(folder_name) != 'finish': log.info('initializing folder sync') # STOPSHIP(emfree): replace by appropriate base class. thread = self.sync_engine_class( self.account_id, folder_name, folder_id_for[folder_name], self.email_address, self.provider_name, self.poll_frequency, self.syncmanager_lock, self.refresh_flags_max, self.retry_fail_classes) thread.start() self.folder_monitors.add(thread) while not self._thread_polling(thread) and \ not self._thread_finished(thread) and \ not thread.ready(): sleep(self.heartbeat) # Allow individual folder sync monitors to shut themselves down # after completing the initial sync. if self._thread_finished(thread) or thread.ready(): log.info('folder sync finished/killed', folder_name=thread.folder_name) # NOTE: Greenlet is automatically removed from the group. self.folder_monitors.join()
def condstore_base_poll(crispin_client, db_session, log, folder_name, shared_state, highestmodseq_fn): """ Base polling logic for IMAP servers which support CONDSTORE and IDLE. The CONDSTORE / HIGHESTMODSEQ mechanism is used to detect new and changed messages that need syncing. """ saved_folder_info = account.get_folder_info(crispin_client.account_id, db_session, folder_name) # Start a session since we're going to IDLE below anyway... # This also resets the folder name cache, which we want in order to # detect folder/label additions and deletions. status = crispin_client.select_folder( folder_name, uidvalidity_cb(db_session, crispin_client.account_id)) log.debug("POLL current modseq: {} | saved modseq: {}".format( status['HIGHESTMODSEQ'], saved_folder_info.highestmodseq)) if status['HIGHESTMODSEQ'] > saved_folder_info.highestmodseq: acc = db_session.query(ImapAccount).get(crispin_client.account_id) save_folder_names(log, acc, crispin_client.folder_names(), db_session) highestmodseq_update(crispin_client, db_session, log, folder_name, saved_folder_info.highestmodseq, highestmodseq_fn, shared_state['syncmanager_lock']) # We really only want to idle on a folder for new messages. Idling on # `All Mail` won't tell us when messages are archived from the Inbox if folder_name.lower() in IDLE_FOLDERS: status = crispin_client.select_folder( folder_name, uidvalidity_cb(db_session, crispin_client.account_id)) idle_frequency = 1800 # 30min log.info("Idling on {0} with {1} timeout".format( folder_name, idle_frequency)) crispin_client.conn.idle() crispin_client.conn.idle_check(timeout=idle_frequency) # If we want to do something with the response, but lousy # because it uses sequence IDs instead of UIDs # resp = c.idle_check(timeout=shared_state['poll_frequency']) # r = dict( EXISTS=[], EXPUNGE=[]) # for msg_uid, cmd in resp: # r[cmd].append(msg_uid) # print r crispin_client.conn.idle_done() log.info("IDLE triggered poll or timeout reached on {0}" .format(folder_name)) else: log.info("Sleeping on {0} for {1} seconds".format( folder_name, shared_state['poll_frequency'])) sleep(shared_state['poll_frequency']) return 'poll'
def sync(self): """ Start per-folder syncs. Only have one per-folder sync in the 'initial' state at a time. """ with mailsync_session_scope() as db_session: with _pool(self.account_id).get() as crispin_client: sync_folders = crispin_client.sync_folders() account = db_session.query(ImapAccount)\ .get(self.account_id) save_folder_names(log, account, crispin_client.folder_names(), db_session) Tag.create_canonical_tags(account.namespace, db_session) folder_id_for = {name: id_ for id_, name in db_session.query( Folder.id, Folder.name).filter_by(account_id=self.account_id)} saved_states = {name: state for name, state in db_session.query(Folder.name, ImapFolderSyncStatus.state) .join(ImapFolderSyncStatus.folder) .filter(ImapFolderSyncStatus.account_id == self.account_id)} for folder_name in sync_folders: if folder_name not in folder_id_for: log.error("Missing Folder object when starting sync", folder_name=folder_name, folder_id_for=folder_id_for) raise MailsyncError("Missing Folder '{}' on account {}" .format(folder_name, self.account_id)) if saved_states.get(folder_name) != 'finish': log.info('initializing folder sync') # STOPSHIP(emfree): replace by appropriate base class. thread = self.sync_engine_class(self.account_id, folder_name, folder_id_for[folder_name], self.email_address, self.provider_name, self.poll_frequency, self.syncmanager_lock, self.refresh_flags_max, self.retry_fail_classes) thread.start() self.folder_monitors.add(thread) while not self._thread_polling(thread) and \ not self._thread_finished(thread) and \ not thread.ready(): sleep(self.heartbeat) # Allow individual folder sync monitors to shut themselves down # after completing the initial sync. if self._thread_finished(thread) or thread.ready(): log.info('folder sync finished/killed', folder_name=thread.folder_name) # NOTE: Greenlet is automatically removed from the group. self.folder_monitors.join()
def test_save_folder_names(db, folder_name_mapping): with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session) saved_folder_names = {name for name, in db_session.query(Folder.name).filter( Folder.account_id == ACCOUNT_ID)} assert saved_folder_names == {'Inbox', '[Gmail]/Spam', '[Gmail]/All Mail', '[Gmail]/Sent Mail', '[Gmail]/Drafts', 'Jobslist', 'Random'}
def sync(self): """ Start per-folder syncs. Only have one per-folder sync in the 'initial' state at a time. """ with session_scope(ignore_soft_deletes=False) as db_session: with _pool(self.account_id).get() as crispin_client: sync_folders = crispin_client.sync_folders() account = db_session.query(ImapAccount)\ .get(self.account_id) save_folder_names(self.log, account, crispin_client.folder_names(), db_session) Tag.create_canonical_tags(account.namespace, db_session) folder_id_for = { name: id_ for id_, name in db_session.query(Folder.id, Folder.name). filter_by(account_id=self.account_id) } saved_states = { name: state for name, state in db_session.query( Folder.name, ImapFolderSyncStatus.state).join( ImapFolderSyncStatus.folder).filter( ImapFolderSyncStatus.account_id == self.account_id) } for folder_name in sync_folders: if folder_name not in folder_id_for: self.log.error("Missing Folder object when starting sync", folder_name=folder_name, folder_id_for=folder_id_for) raise MailsyncError("Missing Folder '{}' on account {}".format( folder_name, self.account_id)) if saved_states.get(folder_name) != 'finish': self.log.info('initializing folder sync') thread = ImapFolderSyncMonitor( self.account_id, folder_name, folder_id_for[folder_name], self.email_address, self.provider_name, self.shared_state, self.folder_state_handlers, self.retry_fail_classes) thread.start() self.folder_monitors.add(thread) while not self._thread_polling(thread) and \ not self._thread_finished(thread): sleep(self.heartbeat) # Allow individual folder sync monitors to shut themselves down # after completing the initial sync. if self._thread_finished(thread): self.log.info('folder sync finished') # NOTE: Greenlet is automatically removed from the group # after finishing. self.folder_monitors.join()
def sync(self): """ Start per-folder syncs. Only have one per-folder sync in the 'initial' state at a time. """ with session_scope(ignore_soft_deletes=False) as db_session: with _pool(self.account_id).get() as crispin_client: sync_folders = crispin_client.sync_folders() account = db_session.query(ImapAccount)\ .get(self.account_id) save_folder_names(self.log, account, crispin_client.folder_names(), db_session) Tag.create_canonical_tags(account.namespace, db_session) folder_id_for = {name: id_ for id_, name in db_session.query( Folder.id, Folder.name).filter_by(account_id=self.account_id)} saved_states = {name: state for name, state in db_session.query(Folder.name, ImapFolderSyncStatus.state) .join(ImapFolderSyncStatus.folder) .filter(ImapFolderSyncStatus.account_id == self.account_id)} for folder_name in sync_folders: if folder_name not in folder_id_for: self.log.error("Missing Folder object when starting sync", folder_name=folder_name, folder_id_for=folder_id_for) raise MailsyncError("Missing Folder '{}' on account {}" .format(folder_name, self.account_id)) if saved_states.get(folder_name) != 'finish': self.log.info('initializing folder sync') thread = ImapFolderSyncMonitor(self.account_id, folder_name, folder_id_for[folder_name], self.email_address, self.provider_name, self.shared_state, self.folder_state_handlers, self.retry_fail_classes) thread.start() self.folder_monitors.add(thread) while not self._thread_polling(thread) and \ not self._thread_finished(thread): sleep(self.heartbeat) # Allow individual folder sync monitors to shut themselves down # after completing the initial sync. if self._thread_finished(thread): self.log.info('folder sync finished') # NOTE: Greenlet is automatically removed from the group # after finishing. self.folder_monitors.join()
def poll_impl(self, crispin_client): log.bind(state='poll') with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info( crispin_client.account_id, db_session, self.folder_name) saved_highestmodseq = saved_folder_info.highestmodseq # Start a session since we're going to IDLE below anyway... # This also resets the folder name cache, which we want in order to # detect folder/label additions and deletions. status = crispin_client.select_folder( self.folder_name, uidvalidity_cb(crispin_client.account_id)) log.debug(current_modseq=status['HIGHESTMODSEQ'], saved_modseq=saved_highestmodseq) if status['HIGHESTMODSEQ'] > saved_highestmodseq: with mailsync_session_scope() as db_session: acc = db_session.query(ImapAccount).get(self.account_id) save_folder_names(log, acc, crispin_client.folder_names(), db_session) self.highestmodseq_update(crispin_client, saved_highestmodseq) # We really only want to idle on a folder for new messages. Idling on # `All Mail` won't tell us when messages are archived from the Inbox if self.folder_name.lower() in IDLE_FOLDERS: status = crispin_client.select_folder( self.folder_name, uidvalidity_cb(crispin_client.account_id)) # Idle doesn't pick up flag changes, so we don't want to idle for # very long, or we won't detect things like messages being marked # as read. idle_frequency = 30 log.info('idling', timeout=idle_frequency) crispin_client.conn.idle() crispin_client.conn.idle_check(timeout=idle_frequency) # If we want to do something with the response, but lousy # because it uses sequence IDs instead of UIDs # resp = c.idle_check(timeout=shared_state['poll_frequency']) # r = dict( EXISTS=[], EXPUNGE=[]) # for msg_uid, cmd in resp: # r[cmd].append(msg_uid) # print r crispin_client.conn.idle_done() log.info('IDLE triggered poll') else: log.info('IDLE sleeping', seconds=self.poll_frequency) sleep(self.poll_frequency)
def test_save_folder_names(db, folder_name_mapping): with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session) saved_folder_names = { name for name, in db_session.query(Folder.name).filter( Folder.account_id == ACCOUNT_ID) } assert saved_folder_names == { 'Inbox', '[Gmail]/Spam', '[Gmail]/All Mail', '[Gmail]/Sent Mail', '[Gmail]/Drafts', 'Jobslist', 'Random' }
def sync(self): """ Start per-folder syncs. Only have one per-folder sync in the 'initial' state at a time. """ with session_scope() as db_session: saved_states = dict() folder_id_for = dict() for saved_state in db_session.query(ImapFolderSyncStatus)\ .filter_by(account_id=self.account_id): saved_states[saved_state.folder.name] = saved_state.state folder_id_for[saved_state.folder.name] = saved_state.folder.id # it's possible we've never started syncs for these folders before for folder_id, folder_name, in \ db_session.query(Folder.id, Folder.name).filter_by( account_id=self.account_id): folder_id_for[folder_name] = folder_id with connection_pool(self.account_id).get() as crispin_client: sync_folders = crispin_client.sync_folders() account = db_session.query(ImapAccount)\ .get(self.account_id) save_folder_names(self.log, account, crispin_client.folder_names(), db_session) Tag.create_canonical_tags(account.namespace, db_session) for folder_name in sync_folders: if saved_states.get(folder_name) != 'finish': self.log.info("Initializing folder sync for {0}" .format(folder_name)) thread = ImapFolderSyncMonitor(self.account_id, folder_name, folder_id_for[folder_name], self.email_address, self.provider, self.shared_state, self.folder_state_handlers, self.retry_fail_classes) thread.start() self.folder_monitors.add(thread) while not self._thread_polling(thread) and \ not self._thread_finished(thread): sleep(self.heartbeat) # Allow individual folder sync monitors to shut themselves down # after completing the initial sync. if self._thread_finished(thread): self.log.info("Folder sync for {} is done." .format(folder_name)) # NOTE: Greenlet is automatically removed from the group # after finishing. self.folder_monitors.join()
def test_save_folder_names(db, default_account, folder_name_mapping): with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, default_account.id, folder_name_mapping, db_session) saved_folder_names = { name for name, in db_session.query(Folder.name).filter(Folder.account_id == default_account.id) } assert saved_folder_names == { "Inbox", "[Gmail]/Spam", "[Gmail]/All Mail", "[Gmail]/Sent Mail", "[Gmail]/Drafts", "Jobslist", "Random", }
def test_folder_delete_cascades_to_tag(db, folder_name_mapping): """Test that when a tag (folder) is deleted, we properly cascade to delete the Tag object too. """ with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session) folders = db_session.query(Folder).filter_by(account_id=ACCOUNT_ID) assert folders.count() == 7 random_folder = folders.filter_by(name='Random').first() assert random_folder is not None random_tag = random_folder.get_associated_tag(db_session) random_tag_id = random_tag.id db.session.commit() folder_name_mapping['extra'] = ['Jobslist'] save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session) db.session.commit() random_tag = db_session.query(Tag).get(random_tag_id) assert random_tag is None
def test_name_collision_folders(db, default_account, folder_name_mapping): # test that when a user-created folder called 'spam' is created, we don't # associate it with the canonical spam tag, but instead give it its own # tag folder_name_mapping['extra'] = ['spam'] with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, default_account.id, folder_name_mapping, db_session) spam_tags = db_session.query(Tag).filter_by( namespace_id=default_account.namespace.id, name='spam') # There should be one 'Gmail/Spam' canonical tag assert spam_tags.count() == 1 assert spam_tags.first().public_id == 'spam' # and one 'imap/spam' non-canonical tag with public_id != 'spam' spam_tags = db_session.query(Tag).filter_by( namespace_id=default_account.namespace.id, name='imap/spam') assert spam_tags.count() == 1 assert spam_tags.first().public_id != 'spam' # test that when a folder called 'spam' is deleted, we don't delete # the canonical 'spam' tag folder_name_mapping['extra'] = [] with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, default_account.id, folder_name_mapping, db_session) spam_tags = db_session.query(Tag).filter_by( namespace_id=default_account.namespace.id, name='spam') # The 'Gmail/Spam' canonical tag should still remain. assert spam_tags.count() == 1 assert spam_tags.first().public_id == 'spam' # The 'imap/spam' non-canonical tag shouldn't spam_tags = db_session.query(Tag).filter_by( namespace_id=default_account.namespace.id, name='imap/spam') assert spam_tags.count() == 0
def test_sync_folder_deletes(db, folder_name_mapping): """Test that folder deletions properly cascade to deletions of ImapFoldSyncStatus and ImapFolderInfo.""" with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session) folders = db_session.query(Folder).filter_by(account_id=ACCOUNT_ID) for folder in folders: add_imap_status_info_rows(folder.id, ACCOUNT_ID, db_session) db_session.commit() assert db_session.query(ImapFolderInfo).count() == 7 assert db_session.query(ImapFolderSyncStatus).count() == 7 folder_name_mapping['extra'] = ['Jobslist'] save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session) saved_folder_names = {name for name, in db_session.query(Folder.name).filter( Folder.account_id == ACCOUNT_ID)} assert saved_folder_names == {'Inbox', '[Gmail]/Spam', '[Gmail]/All Mail', '[Gmail]/Sent Mail', '[Gmail]/Drafts', 'Jobslist'} assert db_session.query(ImapFolderInfo).count() == 6 assert db_session.query(ImapFolderSyncStatus).count() == 6
def test_folder_delete_cascades_to_tag(db, default_account, folder_name_mapping): """Test that when a tag (folder) is deleted, we properly cascade to delete the Tag object too. """ with mailsync_session_scope() as db_session: log = get_logger() save_folder_names(log, default_account.id, folder_name_mapping, db_session) folders = db_session.query(Folder).filter_by( account_id=default_account.id) assert folders.count() == 7 random_folder = folders.filter_by(name='Random').first() assert random_folder is not None random_tag = random_folder.get_associated_tag(db_session) random_tag_id = random_tag.id db.session.commit() folder_name_mapping['extra'] = ['Jobslist'] save_folder_names(log, default_account.id, folder_name_mapping, db_session) db.session.commit() random_tag = db_session.query(Tag).get(random_tag_id) assert random_tag is None
def condstore_base_poll(crispin_client, log, folder_name, shared_state, highestmodseq_fn): """ Base polling logic for IMAP servers which support CONDSTORE and IDLE. The CONDSTORE / HIGHESTMODSEQ mechanism is used to detect new and changed messages that need syncing. """ log.bind(state='poll') with session_scope(ignore_soft_deletes=False) as db_session: saved_folder_info = account.get_folder_info(crispin_client.account_id, db_session, folder_name) saved_highestmodseq = saved_folder_info.highestmodseq # Start a session since we're going to IDLE below anyway... # This also resets the folder name cache, which we want in order to # detect folder/label additions and deletions. status = crispin_client.select_folder( folder_name, uidvalidity_cb(crispin_client.account_id)) log.debug(current_modseq=status['HIGHESTMODSEQ'], saved_modseq=saved_highestmodseq) if status['HIGHESTMODSEQ'] > saved_highestmodseq: with session_scope(ignore_soft_deletes=False) as db_session: acc = db_session.query(ImapAccount).get(crispin_client.account_id) save_folder_names(log, acc, crispin_client.folder_names(), db_session) highestmodseq_update(crispin_client, log, folder_name, saved_highestmodseq, highestmodseq_fn, shared_state['syncmanager_lock']) # We really only want to idle on a folder for new messages. Idling on # `All Mail` won't tell us when messages are archived from the Inbox if folder_name.lower() in IDLE_FOLDERS: status = crispin_client.select_folder( folder_name, uidvalidity_cb(crispin_client.account_id)) idle_frequency = 1800 # 30min log.info('idling', timeout=idle_frequency) crispin_client.conn.idle() crispin_client.conn.idle_check(timeout=idle_frequency) # If we want to do something with the response, but lousy # because it uses sequence IDs instead of UIDs # resp = c.idle_check(timeout=shared_state['poll_frequency']) # r = dict( EXISTS=[], EXPUNGE=[]) # for msg_uid, cmd in resp: # r[cmd].append(msg_uid) # print r crispin_client.conn.idle_done() log.info('IDLE triggered poll') else: log.info('IDLE sleeping', seconds=shared_state['poll_frequency']) sleep(shared_state['poll_frequency']) return 'poll'