def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
remote_uid_count = len(set(crispin_client.all_uids()))
remote_g_metadata = self.__fetch_g_metadata(
crispin_client, local_uids)
remote_uids = sorted(remote_g_metadata.keys(), key=int)
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
self.update_uid_counts(
db_session, remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids))
download_stack = UIDStack()
if self.folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid
# downloaded, we miiight not have the whole thread. This means
# that restarts cause duplicate work, but hopefully these
# folders aren't too huge.
flags = crispin_client.flags(unknown_uids)
for uid in sorted(unknown_uids):
if uid in flags:
download_stack.put(
uid,
GMessage(uid, remote_g_metadata[uid],
flags[uid].flags, flags[uid].labels,
throttled=self.throttled))
change_poller = spawn(self.poll_for_changes, download_stack)
self.__download_queued_threads(crispin_client,
download_stack)
elif self.folder_name in uid_download_folders(crispin_client):
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
for uid in sorted(full_download):
download_stack.put(uid, None)
change_poller = spawn(self.poll_for_changes, download_stack)
self.download_uids(crispin_client, download_stack)
else:
raise MailsyncError(
'Unknown Gmail sync folder: {}'.format(self.folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial
# sync.
rm_cache(remote_g_metadata_cache_file(self.account_id,
self.folder_name))
finally:
if change_poller is not None:
change_poller.kill()
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
remote_uids = sorted(crispin_client.all_uids(), key=int)
remote_uid_count = len(remote_uids)
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
self.remove_deleted_uids(db_session, local_uids,
remote_uids)
unknown_uids = set(remote_uids) - local_uids
self.update_uid_counts(
db_session, remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids))
remote_g_metadata = crispin_client.g_metadata(unknown_uids)
download_stack = UIDStack()
change_poller = spawn(self.poll_for_changes, download_stack)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
if self.is_all_mail(crispin_client):
# Put UIDs on the stack such that UIDs for messages in the
# inbox get downloaded first, and such that higher (i.e., more
# recent) UIDs get downloaded before lower ones.
inbox_uids = crispin_client.search_uids(['X-GM-LABELS inbox'])
inbox_uid_set = set(inbox_uids)
# Note that we have to be checking membership in a /set/ for
# performance.
ordered_uids_to_sync = [u for u in sorted(remote_uids) if u not
in inbox_uid_set] + sorted(inbox_uids)
for uid in ordered_uids_to_sync:
if uid in remote_g_metadata:
metadata = GMetadata(remote_g_metadata[uid].msgid,
remote_g_metadata[uid].thrid,
self.throttled)
download_stack.put(uid, metadata)
self.__download_queued_threads(crispin_client, download_stack)
else:
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
for uid in sorted(full_download):
download_stack.put(uid, None)
self.download_uids(crispin_client, download_stack)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
remote_uids = sorted(crispin_client.all_uids(), key=int)
remote_uid_count = len(remote_uids)
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
self.update_uid_counts(
db_session, remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids))
remote_g_metadata = crispin_client.g_metadata(unknown_uids)
download_stack = UIDStack()
change_poller = spawn(self.poll_for_changes, download_stack)
if self.folder_name in uid_download_folders(crispin_client):
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
for uid in sorted(full_download):
download_stack.put(uid, None)
self.download_uids(crispin_client, download_stack)
elif self.folder_name in thread_expand_folders(crispin_client):
flags = crispin_client.flags(unknown_uids)
for uid in sorted(unknown_uids):
if uid in flags:
gmessage = GMessage(uid, remote_g_metadata[uid],
flags[uid].flags,
flags[uid].labels,
throttled=self.throttled)
download_stack.put(uid, gmessage)
# We always download threads via the 'All Mail' folder.
crispin_client.select_folder(
crispin_client.folder_names()['all'], uidvalidity_cb)
self.__download_queued_threads(crispin_client, download_stack)
else:
raise MailsyncError(
'Unknown Gmail sync folder: {}'.format(self.folder_name))
finally:
if change_poller is not None:
change_poller.kill()
def poll_impl(self):
with self.conn_pool.get() as crispin_client:
download_stack = UIDStack()
self.check_uid_changes(crispin_client,
download_stack,
async_download=False)
if self.should_idle:
self.idle_wait(crispin_client)
# Close IMAP connection before sleeping
if not self.should_idle:
sleep(self.poll_frequency)
def poll_impl(self):
should_idle = False
with self.conn_pool.get() as crispin_client:
crispin_client.select_folder(self.folder_name, uidvalidity_cb)
download_stack = UIDStack()
self.check_uid_changes(crispin_client,
download_stack,
async_download=False)
if self.is_inbox(crispin_client):
# Only idle on the inbox folder
should_idle = True
self.idle_wait(crispin_client)
# Relinquish Crispin connection before sleeping.
if not should_idle:
sleep(self.poll_frequency)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_id)
remote_uids = sorted(crispin_client.all_uids(), key=int)
remote_uid_count = len(remote_uids)
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
self.remove_deleted_uids(db_session, local_uids,
remote_uids)
unknown_uids = set(remote_uids) - local_uids
self.update_uid_counts(
db_session,
remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids))
remote_g_metadata = crispin_client.g_metadata(unknown_uids)
download_stack = UIDStack()
change_poller = spawn(self.poll_for_changes, download_stack)
bind_context(change_poller, 'changepoller', self.account_id,
self.folder_id)
if self.is_all_mail(crispin_client):
# Put UIDs on the stack such that UIDs for messages in the
# inbox get downloaded first, and such that higher (i.e., more
# recent) UIDs get downloaded before lower ones.
inbox_uids = crispin_client.search_uids(['X-GM-LABELS inbox'])
inbox_uid_set = set(inbox_uids)
# Note that we have to be checking membership in a /set/ for
# performance.
ordered_uids_to_sync = [
u for u in sorted(remote_uids) if u not in inbox_uid_set
] + sorted(inbox_uids)
for uid in ordered_uids_to_sync:
if uid in remote_g_metadata:
metadata = GMetadata(remote_g_metadata[uid].msgid,
remote_g_metadata[uid].thrid,
self.throttled)
download_stack.put(uid, metadata)
self.__download_queued_threads(crispin_client, download_stack)
else:
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
for uid in sorted(full_download):
download_stack.put(uid, None)
self.download_uids(crispin_client, download_stack)
finally:
if change_poller is not None:
# schedule change_poller to die
kill(change_poller)
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
remote_uids = sorted(crispin_client.all_uids(), key=int)
remote_uid_count = len(remote_uids)
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
self.update_uid_counts(
db_session,
remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids))
remote_g_metadata = crispin_client.g_metadata(unknown_uids)
download_stack = UIDStack()
change_poller = spawn(self.poll_for_changes, download_stack)
if self.folder_name in uid_download_folders(crispin_client):
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
for uid in sorted(full_download):
download_stack.put(uid, None)
self.download_uids(crispin_client, download_stack)
elif self.folder_name in thread_expand_folders(crispin_client):
flags = crispin_client.flags(unknown_uids)
for uid in sorted(unknown_uids):
if uid in flags:
gmessage = GMessage(uid,
remote_g_metadata[uid],
flags[uid].flags,
flags[uid].labels,
throttled=self.throttled)
download_stack.put(uid, gmessage)
# We always download threads via the 'All Mail' folder.
crispin_client.select_folder(
crispin_client.folder_names()['all'], uidvalidity_cb)
self.__download_queued_threads(crispin_client, download_stack)
else:
raise MailsyncError('Unknown Gmail sync folder: {}'.format(
self.folder_name))
finally:
if change_poller is not None:
change_poller.kill()