def gmail_initial_sync(crispin_client, db_session, log, folder_name,
shared_state, local_uids, uid_download_stack):
remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
folder_name, local_uids,
shared_state['syncmanager_lock'])
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
folder_name))
if folder_name == crispin_client.folder_names()['all']:
log.info("Already have {0} UIDs".format(len(local_uids)))
with shared_state['syncmanager_lock']:
log.debug("gmail_initial_sync grabbed syncmanager_lock")
deleted_uids = remove_deleted_uids(
crispin_client.account_id, db_session, log, folder_name,
local_uids, remote_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
# folders that don't get thread expanded
uid_download_folders = [crispin_client.folder_names()[tag] for tag in
('trash', 'spam', 'all') if tag in
crispin_client.folder_names()]
if folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid downloaded, we
# miiight not have the whole thread. This means that restarts cause
# duplicate work, but hopefully these folders aren't too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
flags[uid].labels))
new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name, log,
message_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack,
shared_state['status_cb'],
shared_state['syncmanager_lock'])
elif folder_name in uid_download_folders:
full_download = deduplicate_message_download(
crispin_client, db_session, log, shared_state['syncmanager_lock'],
remote_g_metadata, unknown_uids)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name,
log, uid_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_uids(crispin_client, db_session, log, folder_name,
uid_download_stack, len(local_uids),
len(remote_uids), shared_state['status_cb'],
shared_state['syncmanager_lock'],
gmail_download_and_commit_uids,
create_gmail_message)
else:
raise MailsyncError(
"Unknown Gmail sync folder: {}".format(folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial sync.
rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
folder_name))
new_uid_poller.kill()
def check_new_g_thrids(account_id, provider, folder_name, log,
message_download_stack, poll_frequency,
syncmanager_lock):
""" Check for new X-GM-THRIDs and add them to the download stack.
We do this by comparing local UID lists to remote UID lists, maintaining
the invariant that (stack uids)+(local uids) == (remote uids).
We also remove local messages that have disappeared from the remote, since
it's totally probable that users will be archiving mail as the initial
sync goes on.
We grab a new IMAP connection from the pool for this to isolate its
actions from whatever the main greenlet may be doing.
Runs until killed. (Intended to be run in a greenlet.)
"""
with connection_pool(account_id).get() as crispin_client:
with session_scope(ignore_soft_deletes=False) as db_session:
crispin_client.select_folder(folder_name,
uidvalidity_cb(
db_session,
crispin_client.account_id))
while True:
log.info("Checking for new/deleted messages during initial sync.")
remote_uids = set(crispin_client.all_uids())
# We lock this section to make sure no messages are being modified
# in the database while we make sure the queue is in a good state.
with syncmanager_lock:
log.debug("check_new_g_thrids acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(account.all_uids(account_id, db_session,
folder_name))
stack_uids = {gm.uid for gm in
message_download_stack.queue}
local_with_pending_uids = local_uids | stack_uids
deleted_uids = remove_deleted_uids(
account_id, db_session, log, folder_name, local_uids,
remote_uids)
log.info("Removed {} deleted UIDs from {}".format(
len(deleted_uids), folder_name))
# filter out messages that have disappeared on the remote side
new_message_download_stack = [gm for gm in
message_download_stack.queue
if gm.uid in remote_uids]
# add in any new uids from the remote
new_uids = [uid for uid in remote_uids if uid not in
local_with_pending_uids]
flags = crispin_client.flags(new_uids)
g_metadata = crispin_client.g_metadata(new_uids)
log.info("Adding {} new messages to the download queue for {}"
.format(min(len(flags), len(g_metadata)), folder_name))
for new_uid in new_uids:
# could have disappeared from the folder in the meantime
if new_uid in flags and new_uid in g_metadata:
new_message_download_stack.append(
GMessage(new_uid, g_metadata[new_uid],
flags[new_uid].flags,
flags[new_uid].labels))
message_download_stack.queue = sorted(
new_message_download_stack, key=lambda m: m.uid)
log.info("Idling on {0} with {1} timeout".format(
folder_name, poll_frequency))
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=poll_frequency)
crispin_client.conn.idle_done()
log.info("IDLE on {0} detected changes or timeout reached"
.format(folder_name))
def check_new_g_thrids(account_id, provider, folder_name, log,
message_download_stack, poll_frequency,
syncmanager_lock):
""" Check for new X-GM-THRIDs and add them to the download stack.
We do this by comparing local UID lists to remote UID lists, maintaining
the invariant that (stack uids)+(local uids) == (remote uids).
We also remove local messages that have disappeared from the remote, since
it's totally probable that users will be archiving mail as the initial
sync goes on.
We grab a new IMAP connection from the pool for this to isolate its
actions from whatever the main greenlet may be doing.
Runs until killed. (Intended to be run in a greenlet.)
"""
with connection_pool(account_id).get() as crispin_client:
with session_scope(ignore_soft_deletes=False) as db_session:
crispin_client.select_folder(folder_name,
uidvalidity_cb(
db_session,
crispin_client.account_id))
while True:
log.info("Checking for new/deleted messages during initial sync.")
remote_uids = set(crispin_client.all_uids())
# We lock this section to make sure no messages are being modified
# in the database while we make sure the queue is in a good state.
with syncmanager_lock:
log.debug("check_new_g_thrids acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(account.all_uids(account_id, db_session,
folder_name))
stack_uids = {gm.uid for gm in
message_download_stack.queue}
local_with_pending_uids = local_uids | stack_uids
deleted_uids = remove_deleted_uids(
account_id, db_session, log, folder_name, local_uids,
remote_uids)
log.info("Removed {} deleted UIDs from {}".format(
len(deleted_uids), folder_name))
# filter out messages that have disappeared on the remote side
new_message_download_stack = [gm for gm in
message_download_stack.queue
if gm.uid in remote_uids]
# add in any new uids from the remote
new_uids = [uid for uid in remote_uids if uid not in
local_with_pending_uids]
flags = crispin_client.flags(new_uids)
g_metadata = crispin_client.g_metadata(new_uids)
log.info("Adding {} new messages to the download queue for {}"
.format(min(len(flags), len(g_metadata)), folder_name))
for new_uid in new_uids:
# could have disappeared from the folder in the meantime
if new_uid in flags and new_uid in g_metadata:
new_message_download_stack.append(
GMessage(new_uid, g_metadata[new_uid],
flags[new_uid].flags,
flags[new_uid].labels))
message_download_stack.queue = sorted(
new_message_download_stack, key=lambda m: m.uid)
log.info("Idling on {0} with {1} timeout".format(
folder_name, poll_frequency))
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=poll_frequency)
crispin_client.conn.idle_done()
log.info("IDLE on {0} detected changes or timeout reached"
.format(folder_name))
def gmail_initial_sync(crispin_client, db_session, log, folder_name,
shared_state, local_uids, uid_download_stack):
remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
folder_name, local_uids,
shared_state['syncmanager_lock'])
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
folder_name))
if folder_name == crispin_client.folder_names()['all']:
log.info("Already have {0} UIDs".format(len(local_uids)))
with shared_state['syncmanager_lock']:
log.debug("gmail_initial_sync grabbed syncmanager_lock")
deleted_uids = remove_deleted_uids(
crispin_client.account_id, db_session, log, folder_name,
local_uids, remote_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
if folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid downloaded, we
# miiight not have the whole thread. This means that restarts cause
# duplicate work, but hopefully these folders aren't too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
flags[uid].labels))
new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name, log,
message_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack,
shared_state['status_cb'],
shared_state['syncmanager_lock'])
elif folder_name in uid_download_folders(crispin_client):
full_download = deduplicate_message_download(
crispin_client, db_session, log, shared_state['syncmanager_lock'],
remote_g_metadata, unknown_uids)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name,
log, uid_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_uids(crispin_client, db_session, log, folder_name,
uid_download_stack, len(local_uids),
len(remote_uids), shared_state['status_cb'],
shared_state['syncmanager_lock'],
gmail_download_and_commit_uids,
create_gmail_message)
else:
raise MailsyncError(
"Unknown Gmail sync folder: {}".format(folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial sync.
rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
folder_name))
new_uid_poller.kill()
def gmail_initial_sync(crispin_client, db_session, log, folder_name,
shared_state, local_uids, uid_download_stack, c):
remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
folder_name, local_uids,
shared_state['syncmanager_lock'], c)
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
folder_name))
if folder_name == crispin_client.folder_names(c)['all']:
log.info("Already have {0} UIDs".format(len(local_uids)))
deleted_uids = remove_deleted_uids(
crispin_client.account_id, db_session, log, folder_name,
local_uids, remote_uids, shared_state['syncmanager_lock'], c)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
if folder_name != crispin_client.folder_names(c)['all']:
# We don't do an initial dedupe for non-All Mail folders because
# we do thread expansion, which means even if we have a given msgid
# downloaded, we miiight not have the whole thread. This means that
# restarts cause duplicate work, but hopefully these folders aren't
# too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids, c)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
flags[uid].labels))
new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name, log,
message_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack,
shared_state['status_cb'],
shared_state['syncmanager_lock'], c)
else:
full_download = deduplicate_message_download(
crispin_client, db_session, log, remote_g_metadata, unknown_uids,
c)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name,
log, uid_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_uids(crispin_client, db_session, log, folder_name,
uid_download_stack, len(local_uids),
len(remote_uids), shared_state['status_cb'],
shared_state['syncmanager_lock'],
gmail_download_and_commit_uids,
account.create_gmail_message, c)
# Complete X-GM-MSGID mapping is no longer needed after initial sync.
rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
folder_name))
new_uid_poller.kill()