def add_new_imapuid(db_session, log, gmessage, folder_name, acc): """ Add ImapUid object for this GMessage if we don't already have one. Parameters ---------- message : GMessage Message to add ImapUid for. folder_name : str Which folder to add the ImapUid in. acc : ImapAccount Which account to associate the message with. (Not looking this up within this function is a db access optimization.) """ if not db_session.query(ImapUid.msg_uid).join(Folder).filter( Folder.name == folder_name, ImapUid.msg_uid == gmessage.uid).all(): message = db_session.query(Message).filter_by( g_msgid=gmessage.g_metadata.msgid).one() new_imapuid = ImapUid( imapaccount=acc, folder=Folder.find_or_create(db_session, acc, folder_name), msg_uid=gmessage.uid, message=message) new_imapuid.update_imap_flags(gmessage.flags, gmessage.labels) db_session.add(new_imapuid) db_session.commit() else: log.debug("Skipping {} imapuid creation for UID {}".format( folder_name, gmessage.uid))
def local_copy(db_session, account, thread_id, from_folder, to_folder): """ Copy thread in the local datastore (*not* the account backend). NOT idempotent. """ if from_folder == to_folder: return with db_write_lock(account.namespace.id): listings = {item.folder.name: item for item in db_session.query(FolderItem).join(Folder).join(Thread) .filter( Thread.namespace_id == account.namespace.id, FolderItem.thread_id == thread_id, Folder.name.in_([from_folder, to_folder])) .all()} if from_folder not in listings: raise LocalActionError("thread {} does not exist in folder {}" .format(thread_id, from_folder)) elif to_folder not in listings: thread = listings[from_folder].thread folder = Folder.find_or_create(db_session, thread.namespace.account, to_folder) thread.folders.add(folder) db_session.commit()
def local_copy(db_session, account, thread_id, from_folder, to_folder): """ Copy thread in the local datastore (*not* the account backend). NOT idempotent. """ if from_folder == to_folder: return with db_write_lock(account.namespace.id): listings = { item.folder.name: item for item in db_session.query(FolderItem).join(Folder).join(Thread). filter(Thread.namespace_id == account.namespace.id, FolderItem.thread_id == thread_id, Folder.name.in_([from_folder, to_folder])).all() } if from_folder not in listings: raise LocalActionError( "thread {} does not exist in folder {}".format( thread_id, from_folder)) elif to_folder not in listings: thread = listings[from_folder].thread folder = Folder.find_or_create(db_session, thread.namespace.account, to_folder) thread.folders.add(folder) db_session.commit()
def create_db_objects(account_id, db_session, log, folder_name, raw_messages, msg_create_fn): new_uids = [] # TODO: Detect which namespace to add message to. (shared folders) # Look up message thread, acc = db_session.query(Account).get(account_id) folder = Folder.find_or_create(db_session, acc, folder_name) for msg in raw_messages: uid = msg_create_fn(db_session, log, acc, folder, msg) if uid is not None: new_uids.append(uid) # imapuid, message, thread, labels return new_uids
def add_new_imapuids(crispin_client, log, db_session, remote_g_metadata, syncmanager_lock, uids): """ Add ImapUid entries only for (already-downloaded) messages. If a message has already been downloaded via another folder, we only need to add `ImapUid` accounting for the current folder. `Message` objects etc. have already been created. """ flags = crispin_client.flags(uids) with syncmanager_lock: log.debug("add_new_imapuids acquired syncmanager_lock") # Since we prioritize download for messages in certain threads, we may # already have ImapUid entries despite calling this method. local_folder_uids = {uid for uid, in db_session.query(ImapUid.msg_uid).join(Folder) .filter( Folder.name == crispin_client.selected_folder_name, ImapUid.msg_uid.in_(uids))} uids = [uid for uid in uids if uid not in local_folder_uids] if uids: # collate message objects to relate the new imapuids imapuid_uid_for = dict([(metadata.msgid, uid) for (uid, metadata) in remote_g_metadata.items() if uid in uids]) imapuid_g_msgids = [remote_g_metadata[uid].msgid for uid in uids] message_for = dict([(imapuid_uid_for[mm.g_msgid], mm) for mm in db_session.query(Message).filter( Message.g_msgid.in_(imapuid_g_msgids))]) acc = db_session.query(ImapAccount).get(crispin_client.account_id) # Folder.find_or_create()'s query will otherwise trigger a flush. with db_session.no_autoflush: new_imapuids = [ImapUid( imapaccount=acc, folder=Folder.find_or_create( db_session, acc, crispin_client.selected_folder_name), msg_uid=uid, message=message_for[uid]) for uid in uids] for item in new_imapuids: item.update_imap_flags(flags[item.msg_uid].flags, flags[item.msg_uid].labels) db_session.add_all(new_imapuids) db_session.commit()
def add_gmail_attrs(db_session, log, new_uid, flags, folder, g_thrid, g_msgid, g_labels, created): """ Gmail-specific post-create-message bits.""" new_uid.message.g_msgid = g_msgid # NOTE: g_thrid == g_msgid on the first message in the thread :) new_uid.message.g_thrid = g_thrid new_uid.update_imap_flags(flags, g_labels) # If we don't disable autoflush here, the thread query may flush a # message to the database with a NULL thread_id, causing a crash. with db_session.no_autoflush: thread = new_uid.message.thread = ImapThread.from_gmail_message( db_session, new_uid.imapaccount.namespace, new_uid.message) # make sure this thread has all the correct labels existing_labels = {folder.name.lower() for folder in thread.folders} # convert things like \Inbox -> Inbox, \Important -> Important new_labels = {l.lstrip('\\') for l in g_labels} | {folder.name} # The IMAP folder name for the inbox on Gmail is INBOX, but there's ALSO a # flag called '\Inbox' on all messages in it... that only appears when you # look at the message with a folder OTHER than INBOX selected. Standardize # on keeping \Inbox in our database. if 'Inbox' in new_labels or 'INBOX' in new_labels: new_labels.discard('INBOX') new_labels.discard('Inbox') new_labels.add('Inbox') # NOTE: Gmail labels are case-insensitive, though we store them in the # original case in the db to not confuse users when displayed. new_labels_ci = {l.lower() for l in new_labels} # Remove labels that have been deleted -- note that the \Inbox, \Sent, # \Important, and \Drafts labels are per-message, not per-thread, but since # we always work at the thread level, _we_ apply the label to the whole # thread. thread.folders = {folder for folder in thread.folders if folder.name.lower() in new_labels_ci or folder.name.lower() in ('inbox', 'sent', 'drafts', 'important')} # add new labels for label in new_labels: if label.lower() not in existing_labels: # The problem here is that Gmail's attempt to squash labels and # IMAP folders into the same abstraction doesn't work perfectly. In # particular, there is a '[Gmail]/Sent' folder, but *also* a 'Sent' # label, and so on. We handle this by only maintaining one folder # object that encapsulates both of these. if label == 'Sent': thread.folders.add(thread.namespace.account.sent_folder) elif label == 'Draft': thread.folders.add(thread.namespace.account.drafts_folder) elif label == 'Starred': thread.folders.add(thread.namespace.account.starred_folder) elif label == 'Important': thread.folders.add(thread.namespace.account.important_folder) else: folder = Folder.find_or_create(db_session, thread.namespace.account, label) thread.folders.add(folder) # Reconciliation for Sent Mail folder: if ('sent' in new_labels_ci and not created and new_uid.message.inbox_uid): if not thread.id: db_session.flush() reconcile_gmail_message(db_session, log, new_uid.message.inbox_uid, new_uid.message, thread.id, g_thrid) return new_uid