def deduplicate_message_download(crispin_client, log, syncmanager_lock, remote_g_metadata, uids): """ Deduplicate message download using X-GM-MSGID. Returns ------- list Deduplicated UIDs. """ with session_scope(ignore_soft_deletes=False) as db_session: local_g_msgids = set(account.g_msgids(crispin_client.account_id, db_session, in_={remote_g_metadata[uid].msgid for uid in uids if uid in remote_g_metadata})) full_download, imapuid_only = partition( lambda uid: uid in remote_g_metadata and remote_g_metadata[uid].msgid in local_g_msgids, sorted(uids, key=int)) if imapuid_only: log.info('skipping already downloaded uids', count=len(imapuid_only)) # Since we always download messages via All Mail and create the # relevant All Mail ImapUids too at that time, we don't need to create # them again here if we're deduping All Mail downloads. if crispin_client.selected_folder_name != \ crispin_client.folder_names()['all']: add_new_imapuids(crispin_client, log, remote_g_metadata, syncmanager_lock, imapuid_only) return full_download
def deduplicate_message_download(crispin_client, db_session, log, syncmanager_lock, remote_g_metadata, uids): """ Deduplicate message download using X-GM-MSGID. Returns ------- list Deduplicated UIDs. """ local_g_msgids = set(account.g_msgids(crispin_client.account_id, db_session, in_={remote_g_metadata[uid].msgid for uid in uids if uid in remote_g_metadata})) full_download, imapuid_only = partition( lambda uid: uid in remote_g_metadata and remote_g_metadata[uid].msgid in local_g_msgids, sorted(uids, key=int)) if imapuid_only: log.info('skipping already downloaded uids', count=len(imapuid_only)) # Since we always download messages via All Mail and create the # relevant All Mail ImapUids too at that time, we don't need to create # them again here if we're deduping All Mail downloads. if crispin_client.selected_folder_name != \ crispin_client.folder_names()['all']: add_new_imapuids(crispin_client, log, db_session, remote_g_metadata, syncmanager_lock, imapuid_only) return full_download
def deduplicate_message_object_creation(account_id, db_session, log, raw_messages): new_g_msgids = {msg.g_msgid for msg in raw_messages} existing_g_msgids = set(account.g_msgids(account_id, db_session, in_=new_g_msgids)) return [msg for msg in raw_messages if msg.g_msgid not in existing_g_msgids]