def test_initial_sync(db, generic_account, inbox_folder, mock_imapclient): # We should really be using hypothesis.given() to generate lots of # different uid sets, but it's not trivial to ensure that no state is # carried over between runs. This will have to suffice for now as a way to # at least establish coverage. uid_dict = uids.example() mock_imapclient.add_folder_data(inbox_folder.name, uid_dict) folder_sync_engine = FolderSyncEngine( generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, "custom", BoundedSemaphore(1), ) folder_sync_engine.initial_sync() saved_uids = db.session.query(ImapUid).filter( ImapUid.folder_id == inbox_folder.id) assert {u.msg_uid for u in saved_uids} == set(uid_dict) saved_message_hashes = {u.message.data_sha256 for u in saved_uids} assert saved_message_hashes == { sha256(v["BODY[]"]).hexdigest() for v in uid_dict.values() }
def test_generic_flags_refresh_expunges_transient_uids(db, generic_account, inbox_folder, mock_imapclient, monkeypatch): # Check that we delete UIDs which are synced but quickly deleted, so never # show up in flags refresh. uid_dict = uids.example() mock_imapclient.add_folder_data(inbox_folder.name, uid_dict) inbox_folder.imapfolderinfo = ImapFolderInfo(account=generic_account, uidvalidity=1, uidnext=1) db.session.commit() folder_sync_engine = FolderSyncEngine(generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1)) folder_sync_engine.initial_sync() # Don't sleep at the end of poll_impl before returning. folder_sync_engine.poll_frequency = 0 folder_sync_engine.poll_impl() msg = db.session.query(Message).filter_by( namespace_id=generic_account.namespace.id).first() transient_uid = ImapUid(folder=inbox_folder, account=generic_account, message=msg, msg_uid=max(uid_dict) + 1) db.session.add(transient_uid) db.session.commit() folder_sync_engine.last_slow_refresh = None folder_sync_engine.poll_impl() with pytest.raises(ObjectDeletedError): transient_uid.id
def test_generic_flags_refresh_expunges_transient_uids( db, generic_account, inbox_folder, mock_imapclient, monkeypatch): # Check that we delete UIDs which are synced but quickly deleted, so never # show up in flags refresh. uid_dict = uids.example() mock_imapclient.add_folder_data(inbox_folder.name, uid_dict) inbox_folder.imapfolderinfo = ImapFolderInfo(account=generic_account, uidvalidity=1, uidnext=1) db.session.commit() folder_sync_engine = FolderSyncEngine(generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1)) folder_sync_engine.initial_sync() # Don't sleep at the end of poll_impl before returning. folder_sync_engine.poll_frequency = 0 folder_sync_engine.poll_impl() msg = db.session.query(Message).filter_by( namespace_id=generic_account.namespace.id).first() transient_uid = ImapUid(folder=inbox_folder, account=generic_account, message=msg, msg_uid=max(uid_dict) + 1) db.session.add(transient_uid) db.session.commit() folder_sync_engine.last_slow_refresh = None folder_sync_engine.poll_impl() with pytest.raises(ObjectDeletedError): transient_uid.id
def test_condstore_flags_refresh(db, default_account, all_mail_folder, mock_imapclient, monkeypatch): monkeypatch.setattr( 'inbox.mailsync.backends.imap.generic.CONDSTORE_FLAGS_REFRESH_BATCH_SIZE', 10) uid_dict = uids.example() mock_imapclient.add_folder_data(all_mail_folder.name, uid_dict) mock_imapclient.capabilities = lambda: ['CONDSTORE'] folder_sync_engine = FolderSyncEngine(default_account.id, default_account.namespace.id, all_mail_folder.name, default_account.email_address, 'gmail', BoundedSemaphore(1)) folder_sync_engine.initial_sync() # Change the labels provided by the mock IMAP server for k, v in mock_imapclient._data[all_mail_folder.name].items(): v['X-GM-LABELS'] = ('newlabel', ) v['MODSEQ'] = (k, ) folder_sync_engine.highestmodseq = 0 # Don't sleep at the end of poll_impl before returning. folder_sync_engine.poll_frequency = 0 folder_sync_engine.poll_impl() imapuids = db.session.query(ImapUid). \ filter_by(folder_id=all_mail_folder.id).all() for imapuid in imapuids: assert 'newlabel' in [l.name for l in imapuid.labels] assert folder_sync_engine.highestmodseq == mock_imapclient.folder_status( all_mail_folder.name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
def sync_engine_stub(db, yahoo_account): db.session.add(Folder(account=yahoo_account, name='Inbox')) db.session.commit() engine = FolderSyncEngine(yahoo_account.id, yahoo_account.namespace.id, "Inbox", TEST_YAHOO_EMAIL, "yahoo", None) return engine
def folder_sync_engine(db, generic_account): db.session.add(Folder(account=generic_account, name='Inbox')) db.session.commit() engine = FolderSyncEngine(generic_account.id, generic_account.namespace.id, "Inbox", generic_account.email_address, generic_account.provider, None) return engine
def test_handle_uidinvalid(db, generic_account, inbox_folder, mock_imapclient): uid_dict = uids.example() mock_imapclient.add_folder_data(inbox_folder.name, uid_dict) inbox_folder.imapfolderinfo = ImapFolderInfo(account=generic_account, uidvalidity=1, uidnext=1) db.session.commit() folder_sync_engine = FolderSyncEngine( generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, "custom", BoundedSemaphore(1), ) folder_sync_engine.initial_sync() mock_imapclient.uidvalidity = 2 with pytest.raises(UidInvalid): folder_sync_engine.poll_impl() new_state = folder_sync_engine.resync_uids() assert new_state == "initial" assert (db.session.query(ImapUid).filter( ImapUid.folder_id == inbox_folder.id).all() == [])
def test_condstore_flags_refresh(db, default_account, all_mail_folder, mock_imapclient, monkeypatch): monkeypatch.setattr( 'inbox.mailsync.backends.imap.generic.CONDSTORE_FLAGS_REFRESH_BATCH_SIZE', 10) uid_dict = uids.example() mock_imapclient.add_folder_data(all_mail_folder.name, uid_dict) mock_imapclient.capabilities = lambda: ['CONDSTORE'] folder_sync_engine = FolderSyncEngine(default_account.id, default_account.namespace.id, all_mail_folder.name, default_account.email_address, 'gmail', BoundedSemaphore(1)) folder_sync_engine.initial_sync() # Change the labels provided by the mock IMAP server for k, v in mock_imapclient._data[all_mail_folder.name].items(): v['X-GM-LABELS'] = ('newlabel',) v['MODSEQ'] = (k,) folder_sync_engine.highestmodseq = 0 # Don't sleep at the end of poll_impl before returning. folder_sync_engine.poll_frequency = 0 folder_sync_engine.poll_impl() imapuids = db.session.query(ImapUid). \ filter_by(folder_id=all_mail_folder.id).all() for imapuid in imapuids: assert 'newlabel' in [l.name for l in imapuid.labels] assert folder_sync_engine.highestmodseq == mock_imapclient.folder_status( all_mail_folder.name, ['HIGHESTMODSEQ'])['HIGHESTMODSEQ']
def test_new_uids_synced_when_polling(db, generic_account, inbox_folder, mock_imapclient): uid_dict = uids.example() mock_imapclient.add_folder_data(inbox_folder.name, uid_dict) inbox_folder.imapfolderinfo = ImapFolderInfo(account=generic_account, uidvalidity=1, uidnext=1) db.session.commit() folder_sync_engine = FolderSyncEngine(generic_account.id, inbox_folder.name, inbox_folder.id, generic_account.email_address, 'custom', BoundedSemaphore(1)) folder_sync_engine.poll_frequency = 0 folder_sync_engine.poll_impl() saved_uids = db.session.query(ImapUid).filter( ImapUid.folder_id == inbox_folder.id) assert {u.msg_uid for u in saved_uids} == set(uid_dict)
def test_imap_message_deduplication(db, generic_account, inbox_folder, generic_trash_folder, mock_imapclient): uid = 22 uid_values = uid_data.example() mock_imapclient.list_folders = lambda: [ (( "\\All", "\\HasNoChildren", ), "/", u"/Inbox"), (( "\\Trash", "\\HasNoChildren", ), "/", u"/Trash"), ] mock_imapclient.idle = lambda: None mock_imapclient.add_folder_data(inbox_folder.name, {uid: uid_values}) mock_imapclient.add_folder_data(generic_trash_folder.name, {uid: uid_values}) folder_sync_engine = FolderSyncEngine( generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, "custom", BoundedSemaphore(1), ) folder_sync_engine.initial_sync() trash_folder_sync_engine = FolderSyncEngine( generic_account.id, generic_account.namespace.id, generic_trash_folder.name, generic_account.email_address, "custom", BoundedSemaphore(1), ) trash_folder_sync_engine.initial_sync() # Check that we have two uids, but just one message. assert [(uid, )] == db.session.query( ImapUid.msg_uid).filter(ImapUid.folder_id == inbox_folder.id).all() assert [(uid, )] == db.session.query(ImapUid.msg_uid).filter( ImapUid.folder_id == generic_trash_folder.id).all() # used to uniquely ID messages body_sha = sha256(uid_values["BODY[]"]).hexdigest() assert (db.session.query(Message).filter( Message.namespace_id == generic_account.namespace.id, Message.data_sha256 == body_sha, ).count() == 1)
def test_new_uids_synced_when_polling(db, generic_account, inbox_folder, mock_imapclient): uid_dict = uids.example() mock_imapclient.add_folder_data(inbox_folder.name, uid_dict) inbox_folder.imapfolderinfo = ImapFolderInfo(account=generic_account, uidvalidity=1, uidnext=1) db.session.commit() folder_sync_engine = FolderSyncEngine(generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1)) folder_sync_engine.poll_frequency = 0 folder_sync_engine.poll_impl() saved_uids = db.session.query(ImapUid).filter( ImapUid.folder_id == inbox_folder.id) assert {u.msg_uid for u in saved_uids} == set(uid_dict)
def test_initial_sync(db, generic_account, inbox_folder, mock_imapclient): # We should really be using hypothesis.given() to generate lots of # different uid sets, but it's not trivial to ensure that no state is # carried over between runs. This will have to suffice for now as a way to # at least establish coverage. uid_dict = uids.example() mock_imapclient.add_folder_data(inbox_folder.name, uid_dict) folder_sync_engine = FolderSyncEngine(generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1)) folder_sync_engine.initial_sync() saved_uids = db.session.query(ImapUid).filter( ImapUid.folder_id == inbox_folder.id) assert {u.msg_uid for u in saved_uids} == set(uid_dict) saved_message_hashes = {u.message.data_sha256 for u in saved_uids} assert saved_message_hashes == {sha256(v['BODY[]']).hexdigest() for v in uid_dict.values()}
def folder_sync_engine(db): from inbox.mailsync.backends.imap.generic import FolderSyncEngine # setup a dummy FolderSyncEngine - we only need to call a couple # methods. email = "*****@*****.**" account = GenericAuthHandler('fastmail').create_account( db.session, email, {"email": email, "password": "******"}) db.session.add(account) db.session.commit() engine = None engine = FolderSyncEngine(account.id, "Inbox", 0, email, "fastmail", None) return engine
def folder_sync_engine(db, monkeypatch): # super ugly, but I don't want to have to mock tons of stuff import inbox.mailsync.backends.imap.generic from inbox.mailsync.backends.imap.generic import FolderSyncEngine monkeypatch.setattr(inbox.mailsync.backends.imap.generic, "_pool", lambda(account): True) # setup a dummy FolderSyncEngine - we only need to call a couple # methods. email = "*****@*****.**" account = GenericAuthHandler('fastmail').create_account( db.session, email, {"email": email, "password": "******"}) db.session.add(account) db.session.commit() engine = None engine = FolderSyncEngine(account.id, "Inbox", 0, email, "fastmail", 3200, None, 20, None) return engine
def initial_sync_impl(self, crispin_client, local_uids, uid_download_stack): with mailsync_session_scope() as db_session: saved_folder_info = common.get_folder_info(self.account_id, db_session, self.folder_name) if saved_folder_info is None: assert (crispin_client.selected_uidvalidity is not None and crispin_client.selected_highestmodseq is not None) common.update_folder_info( crispin_client.account_id, db_session, self.folder_name, crispin_client.selected_uidvalidity, crispin_client.selected_highestmodseq) self.__check_flags(crispin_client, db_session, local_uids) return FolderSyncEngine.initial_sync_impl( self, crispin_client, local_uids, uid_download_stack, spawn_flags_refresh_poller=False)
def test_condstore_flags_refresh(db, default_account, all_mail_folder, mock_imapclient, monkeypatch): monkeypatch.setattr( "inbox.mailsync.backends.imap.generic.CONDSTORE_FLAGS_REFRESH_BATCH_SIZE", 10) uid_dict = uids.example() mock_imapclient.add_folder_data(all_mail_folder.name, uid_dict) mock_imapclient.capabilities = lambda: [b"CONDSTORE"] folder_sync_engine = FolderSyncEngine( default_account.id, default_account.namespace.id, all_mail_folder.name, default_account.email_address, "gmail", BoundedSemaphore(1), ) folder_sync_engine.initial_sync() # Change the labels provided by the mock IMAP server for k, v in mock_imapclient._data[all_mail_folder.name].items(): v[b"X-GM-LABELS"] = (b"newlabel", ) v[b"MODSEQ"] = (k, ) with folder_sync_engine.conn_pool.get() as crispin_client: assert crispin_client.condstore_supported() folder_sync_engine.highestmodseq = 0 # Don't sleep at the end of poll_impl before returning. folder_sync_engine.poll_frequency = 0 folder_sync_engine.poll_impl() imapuids = db.session.query(ImapUid).filter_by( folder_id=all_mail_folder.id).all() for imapuid in imapuids: assert "newlabel" in [l.name for l in imapuid.labels] assert (folder_sync_engine.highestmodseq == mock_imapclient.folder_status( all_mail_folder.name, ["HIGHESTMODSEQ"])[b"HIGHESTMODSEQ"])
def test_imap_message_deduplication(db, generic_account, inbox_folder, generic_trash_folder, mock_imapclient): uid = 22 uid_values = uid_data.example() mock_imapclient.list_folders = lambda: [(('\\All', '\\HasNoChildren',), '/', u'/Inbox'), (('\\Trash', '\\HasNoChildren',), '/', u'/Trash')] mock_imapclient.idle = lambda: None mock_imapclient.add_folder_data(inbox_folder.name, {uid: uid_values}) mock_imapclient.add_folder_data(generic_trash_folder.name, {uid: uid_values}) folder_sync_engine = FolderSyncEngine( generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1), mock.Mock()) folder_sync_engine.initial_sync() trash_folder_sync_engine = FolderSyncEngine( generic_account.id, generic_account.namespace.id, generic_trash_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1), mock.Mock()) trash_folder_sync_engine.initial_sync() # Check that we have two uids, but just one message. assert [(uid,)] == db.session.query(ImapUid.msg_uid).filter( ImapUid.folder_id == inbox_folder.id).all() assert [(uid,)] == db.session.query(ImapUid.msg_uid).filter( ImapUid.folder_id == generic_trash_folder.id).all() # used to uniquely ID messages body_sha = sha256(uid_values['BODY[]']).hexdigest() assert db.session.query(Message).filter( Message.namespace_id == generic_account.namespace.id, Message.data_sha256 == body_sha).count() == 1
def test_handle_uidinvalid(db, generic_account, inbox_folder, mock_imapclient): uid_dict = uids.example() mock_imapclient.add_folder_data(inbox_folder.name, uid_dict) inbox_folder.imapfolderinfo = ImapFolderInfo(account=generic_account, uidvalidity=1, uidnext=1) db.session.commit() folder_sync_engine = FolderSyncEngine(generic_account.id, generic_account.namespace.id, inbox_folder.name, generic_account.email_address, 'custom', BoundedSemaphore(1)) folder_sync_engine.initial_sync() mock_imapclient.uidvalidity = 2 with pytest.raises(UidInvalid): folder_sync_engine.poll_impl() new_state = folder_sync_engine.resync_uids() assert new_state == 'initial' assert db.session.query(ImapUid).filter( ImapUid.folder_id == inbox_folder.id).all() == []
def sync_engine_stub(db, yahoo_account): engine = FolderSyncEngine(yahoo_account.id, yahoo_account.namespace.id, "Inbox", 0, TEST_YAHOO_EMAIL, "yahoo", None) return engine
def __init__(self, *args, **kwargs): FolderSyncEngine.__init__(self, *args, **kwargs) self.folder_id = int(self.folder_id)
def __init__(self, *args, **kwargs): FolderSyncEngine.__init__(self, *args, **kwargs) self.saved_uids = set()
def __init__(self, *args, **kwargs): FolderSyncEngine.__init__(self, *args, **kwargs) self.saved_uids = set()
def __init__(self, *args, **kwargs): FolderSyncEngine.__init__(self, *args, **kwargs) self.folder_id = int(self.folder_id)
def sync_engine_stub(db, yahoo_account): engine = FolderSyncEngine(yahoo_account.id, "Inbox", 0, TEST_YAHOO_EMAIL, "yahoo", 3200, None, 20, []) return engine