def test_update_categories_when_actionlog_entry_missing( db, default_account, message, imapuid ): message.categories_changes = True db.session.commit() update_message_metadata(db.session, imapuid.account, message, False) assert message.categories == {imapuid.folder.category}
def resync_uids_impl(self): # NOTE: first, let's check if the UIVDALIDITY change was spurious, if # it is, just discard it and go on, if it isn't, drop the relevant # entries (filtering by account and folder IDs) from the imapuid table, # download messages, if necessary - in case a message has changed UID - # update UIDs, and discard orphaned messages. -siro with mailsync_session_scope() as db_session: folder_info = db_session.query(ImapFolderInfo). \ filter_by(account_id=self.account_id, folder_id=self.folder_id).one() cached_uidvalidity = folder_info.uidvalidity with self.conn_pool.get() as crispin_client: crispin_client.select_folder(self.folder_name, lambda *args: True) uidvalidity = crispin_client.selected_uidvalidity if uidvalidity <= cached_uidvalidity: log.debug('UIDVALIDITY unchanged') return invalid_uids = db_session.query(ImapUid). \ filter_by(account_id=self.account_id, folder_id=self.folder_id) data_sha256_message = {uid.message.data_sha256: uid.message for uid in invalid_uids} for uid in invalid_uids: db_session.delete(uid) # NOTE: this is necessary (and OK since it doesn't persist any # data) to maintain the order between UIDs deletion and # insertion. Without this, I was seeing constraints violation # on the imapuid table. -siro db_session.flush() remote_uids = crispin_client.all_uids() for remote_uid in remote_uids: raw_message = crispin_client.uids([remote_uid])[0] data_sha256 = sha256(raw_message.body).hexdigest() if data_sha256 in data_sha256_message: message = data_sha256_message[data_sha256] # Create a new imapuid uid = ImapUid(msg_uid=raw_message.uid, message_id=message.id, account_id=self.account_id, folder_id=self.folder_id) uid.update_flags(raw_message.flags) db_session.add(uid) # Update the existing message's metadata too common.update_message_metadata(db_session, uid) del data_sha256_message[data_sha256] else: self.download_and_commit_uids(crispin_client, [remote_uid]) self.heartbeat_status.publish() # FIXME: do we want to throttle the account when recovering # from UIDVALIDITY changes? -siro for message in data_sha256_message.itervalues(): db_session.delete(message) folder_info.uidvalidity = uidvalidity folder_info.highestmodseq = None
def __deduplicate_message_object_creation(self, db_session, raw_messages, account): """ We deduplicate messages based on g_msgid: if we've previously saved a Message object for this raw message, we don't create a new one. But we do create a new ImapUid, associate it to the message, and update flags and categories accordingly. Note: we could do this prior to downloading the actual message body, but that's really more complicated than it's worth. This operation is not super common unless you're regularly moving lots of messages to trash or spam, and even then the overhead of just downloading the body is generally not that high. """ new_g_msgids = {msg.g_msgid for msg in raw_messages} existing_g_msgids = g_msgids(self.namespace_id, db_session, in_=new_g_msgids) brand_new_messages = [ m for m in raw_messages if m.g_msgid not in existing_g_msgids ] previously_synced_messages = [ m for m in raw_messages if m.g_msgid in existing_g_msgids ] if previously_synced_messages: log.info('saving new uids for existing messages', count=len(previously_synced_messages)) account = Account.get(self.account_id, db_session) folder = Folder.get(self.folder_id, db_session) for raw_message in previously_synced_messages: message_obj = db_session.query(Message).filter( Message.namespace_id == self.namespace_id, Message.g_msgid == raw_message.g_msgid).first() if message_obj is None: log.warning('Message disappeared while saving new uid', g_msgid=raw_message.g_msgid, uid=raw_message.uid) brand_new_messages.append(raw_message) continue already_have_uid = ((raw_message.uid, self.folder_id) in {(u.msg_uid, u.folder_id) for u in message_obj.imapuids}) if already_have_uid: log.warning('Skipping existing UID for message', uid=raw_message.uid, message_id=message_obj.id) continue uid = ImapUid(account=account, folder=folder, msg_uid=raw_message.uid, message=message_obj) uid.update_flags(raw_message.flags) uid.update_labels(raw_message.g_labels) common.update_message_metadata(db_session, account, message_obj, uid.is_draft) db_session.commit() return brand_new_messages
def __deduplicate_message_object_creation(self, db_session, raw_messages, account): """ We deduplicate messages based on g_msgid: if we've previously saved a Message object for this raw message, we don't create a new one. But we do create a new ImapUid, associate it to the message, and update flags and categories accordingly. Note: we could do this prior to downloading the actual message body, but that's really more complicated than it's worth. This operation is not super common unless you're regularly moving lots of messages to trash or spam, and even then the overhead of just downloading the body is generally not that high. """ new_g_msgids = {msg.g_msgid for msg in raw_messages} existing_g_msgids = g_msgids(self.namespace_id, db_session, in_=new_g_msgids) brand_new_messages = [m for m in raw_messages if m.g_msgid not in existing_g_msgids] previously_synced_messages = [m for m in raw_messages if m.g_msgid in existing_g_msgids] if previously_synced_messages: log.info('saving new uids for existing messages', count=len(previously_synced_messages)) account = Account.get(self.account_id, db_session) folder = Folder.get(self.folder_id, db_session) for raw_message in previously_synced_messages: message_obj = db_session.query(Message).filter( Message.namespace_id == self.namespace_id, Message.g_msgid == raw_message.g_msgid).first() if message_obj is None: log.warning( 'Message disappeared while saving new uid', g_msgid=raw_message.g_msgid, uid=raw_message.uid) brand_new_messages.append(raw_message) continue already_have_uid = ( (raw_message.uid, self.folder_id) in {(u.msg_uid, u.folder_id) for u in message_obj.imapuids} ) if already_have_uid: log.warning('Skipping existing UID for message', uid=raw_message.uid, message_id=message_obj.id) continue uid = ImapUid(account=account, folder=folder, msg_uid=raw_message.uid, message=message_obj) uid.update_flags(raw_message.flags) uid.update_labels(raw_message.g_labels) common.update_message_metadata( db_session, account, message_obj, uid.is_draft) db_session.commit() return brand_new_messages
def add_custom_label(db, default_account, message): assert len(message.imapuids) == 1 imapuid = message.imapuids[0] existing = [c.name for c in imapuid.categories][0] imapuid.update_labels(["<3"]) db.session.commit() assert set([c.name for c in imapuid.categories]) == set([existing, ""]) update_message_metadata(db.session, default_account, message, False) db.session.commit() return message
def add_inbox_label(db, default_account, message): assert len(message.imapuids) == 1 imapuid = message.imapuids[0] assert set([c.name for c in imapuid.categories]) == set(["all"]) imapuid.update_labels(["\\Inbox"]) db.session.commit() assert set([c.name for c in imapuid.categories]) == set(["all", "inbox"]) update_message_metadata(db.session, default_account, message, False) db.session.commit() return message
def add_custom_label(db, default_account, message): assert len(message.imapuids) == 1 imapuid = message.imapuids[0] existing = [c.name for c in imapuid.categories][0] imapuid.update_labels(['<3']) db.session.commit() assert set([c.name for c in imapuid.categories]) == set([existing, '']) update_message_metadata(db.session, default_account, message, False) db.session.commit() return message
def add_inbox_label(db, default_account, message): assert len(message.imapuids) == 1 imapuid = message.imapuids[0] assert set([c.name for c in imapuid.categories]) == set(['all']) imapuid.update_labels(['\\Inbox']) db.session.commit() assert set([c.name for c in imapuid.categories]) == set(['all', 'inbox']) update_message_metadata(db.session, default_account, message, False) db.session.commit() return message
def folder_and_message_maps(db, default_account): folder_map, message_map = {}, {} for name in ("all", "trash", "spam"): # Create a folder display_name = name.capitalize() if name != "all" else "All Mail" folder = add_fake_folder(db.session, default_account, display_name, name) thread = add_fake_thread(db.session, default_account.namespace.id) # Create a message in the folder message = add_fake_message(db.session, default_account.namespace.id, thread) add_fake_imapuid(db.session, default_account.id, message, folder, 13) update_message_metadata(db.session, default_account, message, False) db.session.commit() folder_map[name] = folder message_map[name] = message return folder_map, message_map
def folder_and_message_maps(db, default_account): folder_map, message_map = {}, {} for name in ('all', 'trash', 'spam'): # Create a folder display_name = name.capitalize() if name != 'all' else 'All Mail' folder = add_fake_folder(db.session, default_account, display_name, name) thread = add_fake_thread(db.session, default_account.namespace.id) # Create a message in the folder message = add_fake_message(db.session, default_account.namespace.id, thread) add_fake_imapuid(db.session, default_account.id, message, folder, 13) update_message_metadata(db.session, default_account, message, False) db.session.commit() folder_map[name] = folder message_map[name] = message return folder_map, message_map
def resync_uids_impl(self): # NOTE: first, let's check if the UIVDALIDITY change was spurious, if # it is, just discard it and go on, if it isn't, drop the relevant # entries (filtering by account and folder IDs) from the imapuid table, # download messages, if necessary - in case a message has changed UID - # update UIDs, and discard orphaned messages. -siro with mailsync_session_scope() as db_session: folder_info = db_session.query(ImapFolderInfo). \ filter_by(account_id=self.account_id, folder_id=self.folder_id).one() cached_uidvalidity = folder_info.uidvalidity with self.conn_pool.get() as crispin_client: crispin_client.select_folder(self.folder_name, lambda *args: True) uidvalidity = crispin_client.selected_uidvalidity if uidvalidity <= cached_uidvalidity: log.debug('UIDVALIDITY unchanged') return invalid_uids = db_session.query(ImapUid). \ filter_by(account_id=self.account_id, folder_id=self.folder_id) data_sha256_message = { uid.message.data_sha256: uid.message for uid in invalid_uids } for uid in invalid_uids: db_session.delete(uid) # NOTE: this is necessary (and OK since it doesn't persist any # data) to maintain the order between UIDs deletion and # insertion. Without this, I was seeing constraints violation # on the imapuid table. -siro db_session.flush() remote_uids = crispin_client.all_uids() for remote_uid in remote_uids: raw_message = crispin_client.uids([remote_uid])[0] data_sha256 = sha256(raw_message.body).hexdigest() if data_sha256 in data_sha256_message: message = data_sha256_message[data_sha256] # Create a new imapuid uid = ImapUid(msg_uid=raw_message.uid, message_id=message.id, account_id=self.account_id, folder_id=self.folder_id) uid.update_flags(raw_message.flags) db_session.add(uid) # Update the existing message's metadata too common.update_message_metadata(db_session, uid) del data_sha256_message[data_sha256] else: self.download_and_commit_uids(crispin_client, [remote_uid]) self.heartbeat_status.publish() # FIXME: do we want to throttle the account when recovering # from UIDVALIDITY changes? -siro for message in data_sha256_message.itervalues(): db_session.delete(message) folder_info.uidvalidity = uidvalidity folder_info.highestmodseq = None
def test_update_categories_when_actionlog_entry_missing(db, default_account, message, imapuid): message.categories_changes = True db.session.commit() update_message_metadata(db.session, imapuid.account, message, False) assert message.categories == {imapuid.folder.category}
def add_new_imapuids(crispin_client, remote_g_metadata, syncmanager_lock, uids): """ Add ImapUid entries only for (already-downloaded) messages. If a message has already been downloaded via another folder, we only need to add `ImapUid` accounting for the current folder. `Message` objects etc. have already been created. """ flags = crispin_client.flags(uids) with syncmanager_lock: with mailsync_session_scope() as db_session: # Since we prioritize download for messages in certain threads, we # may already have ImapUid entries despite calling this method. local_folder_uids = { uid for uid, in db_session.query(ImapUid.msg_uid).join(Folder). filter(ImapUid.account_id == crispin_client.account_id, Folder.name == crispin_client.selected_folder_name, ImapUid.msg_uid.in_(uids)) } uids = [uid for uid in uids if uid not in local_folder_uids] if uids: acc = db_session.query(GmailAccount).get( crispin_client.account_id) # Collate message objects to relate the new imapuids to imapuid_for = dict([ (metadata.msgid, uid) for (uid, metadata) in remote_g_metadata.items() if uid in uids ]) imapuid_g_msgids = [ remote_g_metadata[uid].msgid for uid in uids ] message_for = dict([ (imapuid_for[m.g_msgid], m) for m in db_session.query(Message).join(ImapThread).filter( Message.g_msgid.in_(imapuid_g_msgids), ImapThread.namespace_id == acc.namespace.id) ]) # Stop Folder.find_or_create()'s query from triggering a flush. with db_session.no_autoflush as session: new_imapuids = [ ImapUid(account=acc, folder=Folder.find_or_create( db_session, acc, crispin_client.selected_folder_name), msg_uid=uid, message=message_for[uid]) for uid in uids if uid in message_for ] for uid in new_imapuids: # Skip uids which have disappeared in the meantime if uid.msg_uid in flags: uid.update_flags(flags[uid.msg_uid].flags) uid.update_labels(flags[uid.msg_uid].labels) common.update_message_metadata(session, uid) db_session.add_all(new_imapuids) db_session.commit()
def add_new_imapuids(crispin_client, remote_g_metadata, syncmanager_lock, uids): """ Add ImapUid entries only for (already-downloaded) messages. If a message has already been downloaded via another folder, we only need to add `ImapUid` accounting for the current folder. `Message` objects etc. have already been created. """ flags = crispin_client.flags(uids) with syncmanager_lock: with mailsync_session_scope() as db_session: # Since we prioritize download for messages in certain threads, we # may already have ImapUid entries despite calling this method. local_folder_uids = {uid for uid, in db_session.query(ImapUid.msg_uid).join(Folder) .filter( ImapUid.account_id == crispin_client.account_id, Folder.name == crispin_client.selected_folder_name, ImapUid.msg_uid.in_(uids))} uids = [uid for uid in uids if uid not in local_folder_uids] if uids: acc = db_session.query(GmailAccount).get( crispin_client.account_id) # Collate message objects to relate the new imapuids to imapuid_for = dict([(metadata.msgid, uid) for (uid, metadata) in remote_g_metadata.items() if uid in uids]) imapuid_g_msgids = [remote_g_metadata[uid].msgid for uid in uids] message_for = dict([(imapuid_for[m.g_msgid], m) for m in db_session.query(Message).join(ImapThread) .filter( Message.g_msgid.in_(imapuid_g_msgids), ImapThread.namespace_id == acc.namespace.id)]) # Stop Folder.find_or_create()'s query from triggering a flush. with db_session.no_autoflush as session: new_imapuids = [ImapUid( account=acc, folder=Folder.find_or_create( db_session, acc, crispin_client.selected_folder_name), msg_uid=uid, message=message_for[uid]) for uid in uids if uid in message_for] for uid in new_imapuids: # Skip uids which have disappeared in the meantime if uid.msg_uid in flags: uid.update_flags(flags[uid.msg_uid].flags) uid.update_labels(flags[uid.msg_uid].labels) common.update_message_metadata(session, uid) db_session.add_all(new_imapuids) db_session.commit()