def deduplicate_message_download(crispin_client, log, syncmanager_lock,
remote_g_metadata, uids):
"""
Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
with session_scope(ignore_soft_deletes=False) as db_session:
local_g_msgids = set(account.g_msgids(crispin_client.account_id,
db_session,
in_={remote_g_metadata[uid].msgid
for uid in uids if uid in
remote_g_metadata}))
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and
remote_g_metadata[uid].msgid in local_g_msgids,
sorted(uids, key=int))
if imapuid_only:
log.info('skipping already downloaded uids', count=len(imapuid_only))
# Since we always download messages via All Mail and create the
# relevant All Mail ImapUids too at that time, we don't need to create
# them again here if we're deduping All Mail downloads.
if crispin_client.selected_folder_name != \
crispin_client.folder_names()['all']:
add_new_imapuids(crispin_client, log, remote_g_metadata,
syncmanager_lock, imapuid_only)
return full_download
def deduplicate_message_download(crispin_client, db_session, log,
syncmanager_lock, remote_g_metadata, uids):
"""
Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
local_g_msgids = set(account.g_msgids(crispin_client.account_id,
db_session,
in_={remote_g_metadata[uid].msgid for
uid in uids if uid in
remote_g_metadata}))
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and
remote_g_metadata[uid].msgid in local_g_msgids,
sorted(uids, key=int))
if imapuid_only:
log.info('skipping already downloaded uids', count=len(imapuid_only))
# Since we always download messages via All Mail and create the
# relevant All Mail ImapUids too at that time, we don't need to create
# them again here if we're deduping All Mail downloads.
if crispin_client.selected_folder_name != \
crispin_client.folder_names()['all']:
add_new_imapuids(crispin_client, log, db_session,
remote_g_metadata, syncmanager_lock, imapuid_only)
return full_download
def deduplicate_message_object_creation(account_id, db_session, log,
raw_messages):
new_g_msgids = {msg.g_msgid for msg in raw_messages}
existing_g_msgids = set(account.g_msgids(account_id, db_session,
in_=new_g_msgids))
return [msg for msg in raw_messages if msg.g_msgid not in
existing_g_msgids]
