def create_imap_message(db_session, account, folder, msg): """ IMAP-specific message creation logic. Returns ------- imapuid : inbox.models.backends.imap.ImapUid New db object, which links to new Message and Block objects through relationships. All new objects are uncommitted. """ new_message = Message.create_from_synced( account=account, mid=msg.uid, folder_name=folder.name, received_date=msg.internaldate, body_string=msg.body ) # Check to see if this is a copy of a message that was first created # by the Nylas API. If so, don't create a new object; just use the old one. existing_copy = reconcile_message(new_message, db_session) if existing_copy is not None: new_message = existing_copy imapuid = ImapUid(account=account, folder=folder, msg_uid=msg.uid, message=new_message) imapuid.update_flags(msg.flags) if msg.g_labels is not None: imapuid.update_labels(msg.g_labels) # Update the message's metadata with db_session.no_autoflush: is_draft = imapuid.is_draft and (folder.canonical_name == "drafts" or folder.canonical_name == "all") update_message_metadata(db_session, account, new_message, is_draft) update_contacts_from_message(db_session, new_message, account.namespace) return imapuid
def resync_uids_impl(self): # NOTE: first, let's check if the UIVDALIDITY change was spurious, if # it is, just discard it and go on, if it isn't, drop the relevant # entries (filtering by account and folder IDs) from the imapuid table, # download messages, if necessary - in case a message has changed UID - # update UIDs, and discard orphaned messages. -siro with mailsync_session_scope() as db_session: folder_info = db_session.query(ImapFolderInfo). \ filter_by(account_id=self.account_id, folder_id=self.folder_id).one() cached_uidvalidity = folder_info.uidvalidity with self.conn_pool.get() as crispin_client: crispin_client.select_folder(self.folder_name, lambda *args: True) uidvalidity = crispin_client.selected_uidvalidity if uidvalidity <= cached_uidvalidity: log.debug('UIDVALIDITY unchanged') return invalid_uids = db_session.query(ImapUid). \ filter_by(account_id=self.account_id, folder_id=self.folder_id) data_sha256_message = {uid.message.data_sha256: uid.message for uid in invalid_uids} for uid in invalid_uids: db_session.delete(uid) # NOTE: this is necessary (and OK since it doesn't persist any # data) to maintain the order between UIDs deletion and # insertion. Without this, I was seeing constraints violation # on the imapuid table. -siro db_session.flush() remote_uids = crispin_client.all_uids() for remote_uid in remote_uids: raw_message = crispin_client.uids([remote_uid])[0] data_sha256 = sha256(raw_message.body).hexdigest() if data_sha256 in data_sha256_message: message = data_sha256_message[data_sha256] # Create a new imapuid uid = ImapUid(msg_uid=raw_message.uid, message_id=message.id, account_id=self.account_id, folder_id=self.folder_id) uid.update_flags(raw_message.flags) db_session.add(uid) # Update the existing message's metadata too common.update_message_metadata(db_session, uid) del data_sha256_message[data_sha256] else: self.download_and_commit_uids(crispin_client, [remote_uid]) self.heartbeat_status.publish() # FIXME: do we want to throttle the account when recovering # from UIDVALIDITY changes? -siro for message in data_sha256_message.itervalues(): db_session.delete(message) folder_info.uidvalidity = uidvalidity folder_info.highestmodseq = None
def __deduplicate_message_object_creation(self, db_session, raw_messages, account): """ We deduplicate messages based on g_msgid: if we've previously saved a Message object for this raw message, we don't create a new one. But we do create a new ImapUid, associate it to the message, and update flags and categories accordingly. Note: we could do this prior to downloading the actual message body, but that's really more complicated than it's worth. This operation is not super common unless you're regularly moving lots of messages to trash or spam, and even then the overhead of just downloading the body is generally not that high. """ new_g_msgids = {msg.g_msgid for msg in raw_messages} existing_g_msgids = g_msgids(self.namespace_id, db_session, in_=new_g_msgids) brand_new_messages = [ m for m in raw_messages if m.g_msgid not in existing_g_msgids ] previously_synced_messages = [ m for m in raw_messages if m.g_msgid in existing_g_msgids ] if previously_synced_messages: log.info('saving new uids for existing messages', count=len(previously_synced_messages)) account = Account.get(self.account_id, db_session) folder = Folder.get(self.folder_id, db_session) for raw_message in previously_synced_messages: message_obj = db_session.query(Message).filter( Message.namespace_id == self.namespace_id, Message.g_msgid == raw_message.g_msgid).first() if message_obj is None: log.warning('Message disappeared while saving new uid', g_msgid=raw_message.g_msgid, uid=raw_message.uid) brand_new_messages.append(raw_message) continue already_have_uid = ((raw_message.uid, self.folder_id) in {(u.msg_uid, u.folder_id) for u in message_obj.imapuids}) if already_have_uid: log.warning('Skipping existing UID for message', uid=raw_message.uid, message_id=message_obj.id) continue uid = ImapUid(account=account, folder=folder, msg_uid=raw_message.uid, message=message_obj) uid.update_flags(raw_message.flags) uid.update_labels(raw_message.g_labels) common.update_message_metadata(db_session, account, message_obj, uid.is_draft) db_session.commit() return brand_new_messages
def __deduplicate_message_object_creation(self, db_session, raw_messages, account): """ We deduplicate messages based on g_msgid: if we've previously saved a Message object for this raw message, we don't create a new one. But we do create a new ImapUid, associate it to the message, and update flags and categories accordingly. Note: we could do this prior to downloading the actual message body, but that's really more complicated than it's worth. This operation is not super common unless you're regularly moving lots of messages to trash or spam, and even then the overhead of just downloading the body is generally not that high. """ new_g_msgids = {msg.g_msgid for msg in raw_messages} existing_g_msgids = g_msgids(self.namespace_id, db_session, in_=new_g_msgids) brand_new_messages = [m for m in raw_messages if m.g_msgid not in existing_g_msgids] previously_synced_messages = [m for m in raw_messages if m.g_msgid in existing_g_msgids] if previously_synced_messages: log.info('saving new uids for existing messages', count=len(previously_synced_messages)) account = Account.get(self.account_id, db_session) folder = Folder.get(self.folder_id, db_session) for raw_message in previously_synced_messages: message_obj = db_session.query(Message).filter( Message.namespace_id == self.namespace_id, Message.g_msgid == raw_message.g_msgid).first() if message_obj is None: log.warning( 'Message disappeared while saving new uid', g_msgid=raw_message.g_msgid, uid=raw_message.uid) brand_new_messages.append(raw_message) continue already_have_uid = ( (raw_message.uid, self.folder_id) in {(u.msg_uid, u.folder_id) for u in message_obj.imapuids} ) if already_have_uid: log.warning('Skipping existing UID for message', uid=raw_message.uid, message_id=message_obj.id) continue uid = ImapUid(account=account, folder=folder, msg_uid=raw_message.uid, message=message_obj) uid.update_flags(raw_message.flags) uid.update_labels(raw_message.g_labels) common.update_message_metadata( db_session, account, message_obj, uid.is_draft) db_session.commit() return brand_new_messages
def test_truncate_imapuid_extra_flags(db, default_account, message, folder): imapuid = ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) imapuid.update_flags(['We', 'the', 'People', 'of', 'the', 'United', 'States', 'in', 'Order', 'to', 'form', 'a', 'more', 'perfect', 'Union', 'establish', 'Justice', 'insure', 'domestic', 'Tranquility', 'provide', 'for', 'the', 'common', 'defence', 'promote', 'the', 'general', 'Welfare', 'and', 'secure', 'the', 'Blessings', 'of', 'Liberty', 'to', 'ourselves', 'and', 'our', 'Posterity', 'do', 'ordain', 'and', 'establish', 'this', 'Constitution', 'for', 'the', 'United', 'States', 'of', 'America']) assert len(json.dumps(imapuid.extra_flags)) < 255
def test_truncate_imapuid_extra_flags(db, default_account, message, folder): imapuid = ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) imapuid.update_flags([ 'We', 'the', 'People', 'of', 'the', 'United', 'States', 'in', 'Order', 'to', 'form', 'a', 'more', 'perfect', 'Union', 'establish', 'Justice', 'insure', 'domestic', 'Tranquility', 'provide', 'for', 'the', 'common', 'defence', 'promote', 'the', 'general', 'Welfare', 'and', 'secure', 'the', 'Blessings', 'of', 'Liberty', 'to', 'ourselves', 'and', 'our', 'Posterity', 'do', 'ordain', 'and', 'establish', 'this', 'Constitution', 'for', 'the', 'United', 'States', 'of', 'America' ]) assert len(json.dumps(imapuid.extra_flags)) < 255
def create_imap_message(db_session, account, folder, msg): """ IMAP-specific message creation logic. Returns ------- imapuid : inbox.models.backends.imap.ImapUid New db object, which links to new Message and Block objects through relationships. All new objects are uncommitted. """ log.debug("creating message", account_id=account.id, folder_name=folder.name, mid=msg.uid) new_message = Message.create_from_synced( account=account, mid=msg.uid, folder_name=folder.name, received_date=msg.internaldate, body_string=msg.body, ) # Check to see if this is a copy of a message that was first created # by the Nylas API. If so, don't create a new object; just use the old one. existing_copy = reconcile_message(new_message, db_session) if existing_copy is not None: new_message = existing_copy imapuid = ImapUid(account=account, folder=folder, msg_uid=msg.uid, message=new_message) imapuid.update_flags(msg.flags) if msg.g_labels is not None: imapuid.update_labels(msg.g_labels) # Update the message's metadata with db_session.no_autoflush: is_draft = imapuid.is_draft and (folder.canonical_name == "drafts" or folder.canonical_name == "all") update_message_metadata(db_session, account, new_message, is_draft) update_contacts_from_message(db_session, new_message, account.namespace.id) return imapuid
def create_imap_message(db_session, log, account, folder, msg): """ IMAP-specific message creation logic. This is the one function in this file that gets to take an account object instead of an account_id, because we need to relate the account to ImapUids for versioning to work, since it needs to look up the namespace. Returns ------- imapuid : inbox.models.tables.imap.ImapUid New db object, which links to new Message and Block objects through relationships. All new objects are uncommitted. """ new_message = Message.create_from_synced(account=account, mid=msg.uid, folder_name=folder.name, received_date=msg.internaldate, body_string=msg.body) # Check to see if this is a copy of a message that was first created # by the Inbox API. If so, don't create a new object; just use the old one. existing_copy = reconcile_message(new_message, db_session) if existing_copy is not None: new_message = existing_copy imapuid = ImapUid(account=account, folder=folder, msg_uid=msg.uid, message=new_message) imapuid.update_flags(msg.flags) if msg.g_labels is not None: imapuid.update_labels(msg.g_labels) # Update the message's metadata with db_session.no_autoflush: update_message_metadata(db_session, account, new_message, imapuid.is_draft) update_contacts_from_message(db_session, new_message, account.namespace) return imapuid
def resync_uids_impl(self): # NOTE: first, let's check if the UIVDALIDITY change was spurious, if # it is, just discard it and go on, if it isn't, drop the relevant # entries (filtering by account and folder IDs) from the imapuid table, # download messages, if necessary - in case a message has changed UID - # update UIDs, and discard orphaned messages. -siro with mailsync_session_scope() as db_session: folder_info = db_session.query(ImapFolderInfo). \ filter_by(account_id=self.account_id, folder_id=self.folder_id).one() cached_uidvalidity = folder_info.uidvalidity with self.conn_pool.get() as crispin_client: crispin_client.select_folder(self.folder_name, lambda *args: True) uidvalidity = crispin_client.selected_uidvalidity if uidvalidity <= cached_uidvalidity: log.debug('UIDVALIDITY unchanged') return invalid_uids = db_session.query(ImapUid). \ filter_by(account_id=self.account_id, folder_id=self.folder_id) data_sha256_message = { uid.message.data_sha256: uid.message for uid in invalid_uids } for uid in invalid_uids: db_session.delete(uid) # NOTE: this is necessary (and OK since it doesn't persist any # data) to maintain the order between UIDs deletion and # insertion. Without this, I was seeing constraints violation # on the imapuid table. -siro db_session.flush() remote_uids = crispin_client.all_uids() for remote_uid in remote_uids: raw_message = crispin_client.uids([remote_uid])[0] data_sha256 = sha256(raw_message.body).hexdigest() if data_sha256 in data_sha256_message: message = data_sha256_message[data_sha256] # Create a new imapuid uid = ImapUid(msg_uid=raw_message.uid, message_id=message.id, account_id=self.account_id, folder_id=self.folder_id) uid.update_flags(raw_message.flags) db_session.add(uid) # Update the existing message's metadata too common.update_message_metadata(db_session, uid) del data_sha256_message[data_sha256] else: self.download_and_commit_uids(crispin_client, [remote_uid]) self.heartbeat_status.publish() # FIXME: do we want to throttle the account when recovering # from UIDVALIDITY changes? -siro for message in data_sha256_message.itervalues(): db_session.delete(message) folder_info.uidvalidity = uidvalidity folder_info.highestmodseq = None
def test_truncate_imapuid_extra_flags(db, default_account, message, folder): imapuid = ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) imapuid.update_flags( [ "We", "the", "People", "of", "the", "United", "States", "in", "Order", "to", "form", "a", "more", "perfect", "Union", "establish", "Justice", "insure", "domestic", "Tranquility", "provide", "for", "the", "common", "defence", "promote", "the", "general", "Welfare", "and", "secure", "the", "Blessings", "of", "Liberty", "to", "ourselves", "and", "our", "Posterity", "do", "ordain", "and", "establish", "this", "Constitution", "for", "the", "United", "States", "of", "America", ] ) assert len(json.dumps(imapuid.extra_flags)) < 255
def test_truncate_imapuid_extra_flags(db, default_account, message, folder): imapuid = ImapUid(message=message, account_id=default_account.id, msg_uid=2222, folder=folder) imapuid.update_flags([ "We", "the", "People", "of", "the", "United", "States", "in", "Order", "to", "form", "a", "more", "perfect", "Union", "establish", "Justice", "insure", "domestic", "Tranquility", "provide", "for", "the", "common", "defence", "promote", "the", "general", "Welfare", "and", "secure", "the", "Blessings", "of", "Liberty", "to", "ourselves", "and", "our", "Posterity", "do", "ordain", "and", "establish", "this", "Constitution", "for", "the", "United", "States", "of", "America", ]) assert len(json.dumps(imapuid.extra_flags)) < 255