def deduplicate_message_download(crispin_client, db_session, log,
syncmanager_lock, remote_g_metadata, uids):
""" Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
local_g_msgids = set(account.g_msgids(crispin_client.account_id,
db_session,
in_=[remote_g_metadata[uid].msgid for
uid in uids if uid in
remote_g_metadata]))
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and
remote_g_metadata[uid].msgid in local_g_msgids,
sorted(uids, key=int))
if imapuid_only:
log.info("Skipping {0} uids already downloaded"
.format(len(imapuid_only)))
# Since we always download messages via All Mail and create the
# relevant All Mail ImapUids too at that time, we don't need to create
# them again here if we're deduping All Mail downloads.
if crispin_client.selected_folder_name != \
crispin_client.folder_names()['all']:
add_new_imapuids(crispin_client, log, db_session,
remote_g_metadata, syncmanager_lock, imapuid_only)
return full_download
def deduplicate_message_download(crispin_client, db_session, log,
syncmanager_lock, remote_g_metadata, uids):
""" Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
local_g_msgids = set(account.g_msgids(crispin_client.account_id,
db_session,
in_=[remote_g_metadata[uid].msgid for
uid in uids if uid in
remote_g_metadata]))
full_download, imapuid_only = partition(
lambda uid: uid in remote_g_metadata and
remote_g_metadata[uid].msgid in local_g_msgids,
sorted(uids, key=int))
if imapuid_only:
log.info("Skipping {0} uids already downloaded"
.format(len(imapuid_only)))
# Since we always download messages via All Mail and create the
# relevant All Mail ImapUids too at that time, we don't need to create
# them again here if we're deduping All Mail downloads.
if crispin_client.selected_folder_name != \
crispin_client.folder_names()['all']:
add_new_imapuids(crispin_client, log, db_session,
remote_g_metadata, syncmanager_lock, imapuid_only)
return full_download
def deduplicate_message_object_creation(account_id, db_session, log,
raw_messages):
log.info("Deduplicating message object creation.")
new_g_msgids = {msg.g_msgid for msg in raw_messages}
existing_g_msgids = set(account.g_msgids(account_id, db_session,
in_=new_g_msgids))
return [msg for msg in raw_messages if msg.g_msgid not in
existing_g_msgids]
def deduplicate_message_object_creation(account_id, db_session, log,
raw_messages):
log.info("Deduplicating message object creation.")
new_g_msgids = {msg.g_msgid for msg in raw_messages}
existing_g_msgids = set(account.g_msgids(account_id, db_session,
in_=new_g_msgids))
return [msg for msg in raw_messages if msg.g_msgid not in
existing_g_msgids]
def deduplicate_message_download(crispin_client, db_session, log,
remote_g_metadata, uids, c):
""" Deduplicate message download using X-GM-MSGID.
Returns
-------
list
Deduplicated UIDs.
"""
local_g_msgids = set(account.g_msgids(crispin_client.account_id,
db_session,
in_=[remote_g_metadata[uid].msgid for
uid in uids]))
full_download, imapuid_only = partition(lambda uid:
remote_g_metadata[uid].msgid in
local_g_msgids, sorted(uids,
key=int))
if imapuid_only:
log.info("Skipping {0} uids already downloaded"
.format(len(imapuid_only)))
add_new_imapuids(crispin_client, db_session, remote_g_metadata,
imapuid_only, c)
return full_download
