def gmail_initial_sync(crispin_client, db_session, log, folder_name,
shared_state, local_uids, c):
remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
folder_name, local_uids, c)
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
folder_name))
if folder_name == crispin_client.folder_names(c)['All']:
log.info("Already have {0} UIDs".format(len(local_uids)))
local_uids = set(local_uids).difference(
remove_deleted_uids(crispin_client.account_id,
db_session, log, folder_name, local_uids, remote_uids, c))
unknown_uids = set(remote_uids).difference(set(local_uids))
if folder_name != crispin_client.folder_names(c)['All']:
chunked_thread_download(crispin_client, db_session, log, folder_name,
remote_g_metadata, unknown_uids, shared_state['status_cb'],
shared_state['syncmanager_lock'], c)
else:
full_download = deduplicate_message_download(crispin_client,
db_session, log, remote_g_metadata, unknown_uids, c)
chunked_uid_download(crispin_client, db_session, log, folder_name,
full_download, len(local_uids), len(remote_uids),
shared_state['status_cb'], shared_state['syncmanager_lock'],
gmail_download_and_commit_uids,
account.create_gmail_message, c)
# Complete X-GM-MSGID mapping is no longer needed after initial sync.
rm_cache(remote_g_metadata_cache_file(crispin_client.account_id, folder_name))
def initial_sync_impl(self, crispin_client):
# We wrap the block in a try/finally because the greenlets like
# change_poller need to be killed when this greenlet is interrupted
change_poller = None
try:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
remote_uid_count = len(set(crispin_client.all_uids()))
remote_g_metadata = self.__fetch_g_metadata(
crispin_client, local_uids)
remote_uids = sorted(remote_g_metadata.keys(), key=int)
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
self.update_uid_counts(
db_session, remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids))
download_stack = UIDStack()
if self.folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid
# downloaded, we miiight not have the whole thread. This means
# that restarts cause duplicate work, but hopefully these
# folders aren't too huge.
flags = crispin_client.flags(unknown_uids)
for uid in sorted(unknown_uids):
if uid in flags:
download_stack.put(
uid,
GMessage(uid, remote_g_metadata[uid],
flags[uid].flags, flags[uid].labels,
throttled=self.throttled))
change_poller = spawn(self.poll_for_changes, download_stack)
self.__download_queued_threads(crispin_client,
download_stack)
elif self.folder_name in uid_download_folders(crispin_client):
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
for uid in sorted(full_download):
download_stack.put(uid, None)
change_poller = spawn(self.poll_for_changes, download_stack)
self.download_uids(crispin_client, download_stack)
else:
raise MailsyncError(
'Unknown Gmail sync folder: {}'.format(self.folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial
# sync.
rm_cache(remote_g_metadata_cache_file(self.account_id,
self.folder_name))
finally:
if change_poller is not None:
change_poller.kill()
def gmail_initial_sync(crispin_client, db_session, log, folder_name,
shared_state, local_uids, uid_download_stack):
remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
folder_name, local_uids,
shared_state['syncmanager_lock'])
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
folder_name))
if folder_name == crispin_client.folder_names()['all']:
log.info("Already have {0} UIDs".format(len(local_uids)))
with shared_state['syncmanager_lock']:
log.debug("gmail_initial_sync grabbed syncmanager_lock")
deleted_uids = remove_deleted_uids(
crispin_client.account_id, db_session, log, folder_name,
local_uids, remote_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
# folders that don't get thread expanded
uid_download_folders = [crispin_client.folder_names()[tag] for tag in
('trash', 'spam', 'all') if tag in
crispin_client.folder_names()]
if folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid downloaded, we
# miiight not have the whole thread. This means that restarts cause
# duplicate work, but hopefully these folders aren't too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
flags[uid].labels))
new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name, log,
message_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack,
shared_state['status_cb'],
shared_state['syncmanager_lock'])
elif folder_name in uid_download_folders:
full_download = deduplicate_message_download(
crispin_client, db_session, log, shared_state['syncmanager_lock'],
remote_g_metadata, unknown_uids)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name,
log, uid_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_uids(crispin_client, db_session, log, folder_name,
uid_download_stack, len(local_uids),
len(remote_uids), shared_state['status_cb'],
shared_state['syncmanager_lock'],
gmail_download_and_commit_uids,
create_gmail_message)
else:
raise MailsyncError(
"Unknown Gmail sync folder: {}".format(folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial sync.
rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
folder_name))
new_uid_poller.kill()
def gmail_initial_sync(crispin_client, db_session, log, folder_name,
shared_state, local_uids, uid_download_stack,
msg_create_fn):
remote_uid_count = len(set(crispin_client.all_uids()))
remote_g_metadata, sync_info = get_g_metadata(
crispin_client, db_session, log, folder_name, local_uids,
shared_state['syncmanager_lock'])
sync_type, update_uid_count = sync_info
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info(remote_uid_count=len(remote_uids))
if folder_name == crispin_client.folder_names()['all']:
log.info(local_uid_count=len(local_uids))
with shared_state['syncmanager_lock']:
log.debug('gmail_initial_sync grabbed syncmanager_lock')
deleted_uids = remove_deleted_uids(
crispin_client.account_id, db_session, log, folder_name,
local_uids, remote_uids)
delete_uid_count = len(deleted_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
# Persist the num(messages) to sync (any type of sync: download,
# update or delete) before we start.
# Note that num_local_deleted, num_local_updated ARE the numbers to
# delete/update too since we make those changes rightaway before we start
# downloading messages.
update_uid_counts(db_session, log, crispin_client.account_id, folder_name,
remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids),
update_uid_count=update_uid_count,
delete_uid_count=delete_uid_count,
sync_type=sync_type)
if folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid downloaded, we
# miiight not have the whole thread. This means that restarts cause
# duplicate work, but hopefully these folders aren't too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
flags[uid].labels))
new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name, log,
message_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack,
shared_state['syncmanager_lock'])
elif folder_name in uid_download_folders(crispin_client):
full_download = deduplicate_message_download(
crispin_client, db_session, log, shared_state['syncmanager_lock'],
remote_g_metadata, unknown_uids)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name,
log, uid_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_uids(crispin_client, db_session, log, folder_name,
uid_download_stack, len(local_uids),
len(unknown_uids),
shared_state['syncmanager_lock'],
gmail_download_and_commit_uids, msg_create_fn)
else:
raise MailsyncError(
'Unknown Gmail sync folder: {}'.format(folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial sync.
rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
folder_name))
new_uid_poller.kill()
def initial_sync_impl(self, crispin_client, local_uids,
uid_download_stack):
# We wrap the block in a try/finally because the greenlets like
# new_uid_poller need to be killed when this greenlet is interrupted
try:
remote_uid_count = len(set(crispin_client.all_uids()))
remote_g_metadata, update_uid_count = self.__fetch_g_metadata(
crispin_client, local_uids)
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info(remote_uid_count=len(remote_uids))
if self.folder_name == crispin_client.folder_names()['all']:
log.info(local_uid_count=len(local_uids))
with self.syncmanager_lock:
log.debug('gmail_initial_sync grabbed syncmanager_lock')
with mailsync_session_scope() as db_session:
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
delete_uid_count = len(deleted_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
# Persist the num(messages) to sync (any type of sync:
# download, update or delete) before we start. Note that
# num_local_deleted, num_local_updated ARE the numbers to
# delete/update too since we make those changes rightaway
# before we start downloading messages.
self.update_uid_counts(
db_session,
remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids),
update_uid_count=update_uid_count,
delete_uid_count=delete_uid_count)
if self.folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid
# downloaded, we miiight not have the whole thread. This means
# that restarts cause duplicate work, but hopefully these
# folders aren't too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid],
flags[uid].flags, flags[uid].labels))
new_uid_poller = spawn(self.__check_new_g_thrids,
message_download_stack)
self.__download_queued_threads(crispin_client,
message_download_stack)
elif self.folder_name in uid_download_folders(crispin_client):
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(self.check_new_uids, uid_download_stack)
self.download_uids(crispin_client, uid_download_stack)
else:
raise MailsyncError('Unknown Gmail sync folder: {}'.format(
self.folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial
# sync.
rm_cache(
remote_g_metadata_cache_file(self.account_id,
self.folder_name))
finally:
new_uid_poller.kill()
def initial_sync_impl(self, crispin_client, local_uids,
uid_download_stack):
# We wrap the block in a try/finally because the greenlets like
# new_uid_poller need to be killed when this greenlet is interrupted
try:
remote_uid_count = len(set(crispin_client.all_uids()))
remote_g_metadata, update_uid_count = self.__fetch_g_metadata(
crispin_client, local_uids)
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info(remote_uid_count=len(remote_uids))
if self.folder_name == crispin_client.folder_names()['all']:
log.info(local_uid_count=len(local_uids))
with self.syncmanager_lock:
log.debug('gmail_initial_sync grabbed syncmanager_lock')
with mailsync_session_scope() as db_session:
deleted_uids = self.remove_deleted_uids(
db_session, local_uids, remote_uids)
delete_uid_count = len(deleted_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
# Persist the num(messages) to sync (any type of sync:
# download, update or delete) before we start. Note that
# num_local_deleted, num_local_updated ARE the numbers to
# delete/update too since we make those changes rightaway
# before we start downloading messages.
self.update_uid_counts(
db_session, remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids),
update_uid_count=update_uid_count,
delete_uid_count=delete_uid_count)
if self.folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid
# downloaded, we miiight not have the whole thread. This means
# that restarts cause duplicate work, but hopefully these
# folders aren't too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid],
flags[uid].flags, flags[uid].labels))
new_uid_poller = spawn(self.__check_new_g_thrids,
message_download_stack)
self.__download_queued_threads(crispin_client,
message_download_stack)
elif self.folder_name in uid_download_folders(crispin_client):
full_download = self.__deduplicate_message_download(
crispin_client, remote_g_metadata, unknown_uids)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(self.check_new_uids, uid_download_stack)
self.download_uids(crispin_client, uid_download_stack)
else:
raise MailsyncError(
'Unknown Gmail sync folder: {}'.format(self.folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial
# sync.
rm_cache(remote_g_metadata_cache_file(self.account_id,
self.folder_name))
finally:
new_uid_poller.kill()
def gmail_initial_sync(crispin_client, log, folder_name, shared_state,
local_uids, uid_download_stack, msg_create_fn):
remote_uid_count = len(set(crispin_client.all_uids()))
remote_g_metadata, update_uid_count = get_g_metadata(
crispin_client, log, folder_name, local_uids,
shared_state['syncmanager_lock'])
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info(remote_uid_count=len(remote_uids))
if folder_name == crispin_client.folder_names()['all']:
log.info(local_uid_count=len(local_uids))
with shared_state['syncmanager_lock']:
log.debug('gmail_initial_sync grabbed syncmanager_lock')
with session_scope(ignore_soft_deletes=False) as db_session:
deleted_uids = remove_deleted_uids(
crispin_client.account_id, db_session, log, folder_name,
local_uids, remote_uids)
delete_uid_count = len(deleted_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
# Persist the num(messages) to sync (any type of sync: download,
# update or delete) before we start.
# Note that num_local_deleted, num_local_updated ARE the numbers to
# delete/update too since we make those changes rightaway before we
# start downloading messages.
update_uid_counts(db_session, log, crispin_client.account_id,
folder_name, remote_uid_count=remote_uid_count,
download_uid_count=len(unknown_uids),
update_uid_count=update_uid_count,
delete_uid_count=delete_uid_count)
if folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid downloaded, we
# miiight not have the whole thread. This means that restarts cause
# duplicate work, but hopefully these folders aren't too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
flags[uid].labels))
new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name, log,
message_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_threads(crispin_client, log, folder_name,
message_download_stack,
shared_state['syncmanager_lock'])
elif folder_name in uid_download_folders(crispin_client):
full_download = deduplicate_message_download(
crispin_client, log, shared_state['syncmanager_lock'],
remote_g_metadata, unknown_uids)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
folder_name,
log, uid_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_uids(crispin_client, log, folder_name,
uid_download_stack, len(local_uids),
len(unknown_uids),
shared_state['syncmanager_lock'],
gmail_download_and_commit_uids, msg_create_fn)
else:
raise MailsyncError(
'Unknown Gmail sync folder: {}'.format(folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial sync.
rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
folder_name))
new_uid_poller.kill()
def gmail_initial_sync(crispin_client, db_session, log, folder_name,
shared_state, local_uids, uid_download_stack):
remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
folder_name, local_uids,
shared_state['syncmanager_lock'])
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
folder_name))
if folder_name == crispin_client.folder_names()['all']:
log.info("Already have {0} UIDs".format(len(local_uids)))
with shared_state['syncmanager_lock']:
log.debug("gmail_initial_sync grabbed syncmanager_lock")
deleted_uids = remove_deleted_uids(
crispin_client.account_id, db_session, log, folder_name,
local_uids, remote_uids)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
if folder_name == crispin_client.folder_names()['inbox']:
# We don't do an initial dedupe for Inbox because we do thread
# expansion, which means even if we have a given msgid downloaded, we
# miiight not have the whole thread. This means that restarts cause
# duplicate work, but hopefully these folders aren't too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
flags[uid].labels))
new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name, log,
message_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack,
shared_state['status_cb'],
shared_state['syncmanager_lock'])
elif folder_name in uid_download_folders(crispin_client):
full_download = deduplicate_message_download(
crispin_client, db_session, log, shared_state['syncmanager_lock'],
remote_g_metadata, unknown_uids)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name,
log, uid_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_uids(crispin_client, db_session, log, folder_name,
uid_download_stack, len(local_uids),
len(remote_uids), shared_state['status_cb'],
shared_state['syncmanager_lock'],
gmail_download_and_commit_uids,
create_gmail_message)
else:
raise MailsyncError(
"Unknown Gmail sync folder: {}".format(folder_name))
# Complete X-GM-MSGID mapping is no longer needed after initial sync.
rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
folder_name))
new_uid_poller.kill()
def gmail_initial_sync(crispin_client, db_session, log, folder_name,
shared_state, local_uids, uid_download_stack, c):
remote_g_metadata = get_g_metadata(crispin_client, db_session, log,
folder_name, local_uids,
shared_state['syncmanager_lock'], c)
remote_uids = sorted(remote_g_metadata.keys(), key=int)
log.info("Found {0} UIDs for folder {1}".format(len(remote_uids),
folder_name))
if folder_name == crispin_client.folder_names(c)['all']:
log.info("Already have {0} UIDs".format(len(local_uids)))
deleted_uids = remove_deleted_uids(
crispin_client.account_id, db_session, log, folder_name,
local_uids, remote_uids, shared_state['syncmanager_lock'], c)
local_uids = set(local_uids) - deleted_uids
unknown_uids = set(remote_uids) - local_uids
if folder_name != crispin_client.folder_names(c)['all']:
# We don't do an initial dedupe for non-All Mail folders because
# we do thread expansion, which means even if we have a given msgid
# downloaded, we miiight not have the whole thread. This means that
# restarts cause duplicate work, but hopefully these folders aren't
# too huge.
message_download_stack = LifoQueue()
flags = crispin_client.flags(unknown_uids, c)
for uid in unknown_uids:
if uid in flags:
message_download_stack.put(
GMessage(uid, remote_g_metadata[uid], flags[uid].flags,
flags[uid].labels))
new_uid_poller = spawn(check_new_g_thrids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name, log,
message_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack,
shared_state['status_cb'],
shared_state['syncmanager_lock'], c)
else:
full_download = deduplicate_message_download(
crispin_client, db_session, log, remote_g_metadata, unknown_uids,
c)
add_uids_to_stack(full_download, uid_download_stack)
new_uid_poller = spawn(check_new_uids, crispin_client.account_id,
crispin_client.PROVIDER, folder_name,
log, uid_download_stack,
shared_state['poll_frequency'],
shared_state['syncmanager_lock'])
download_queued_uids(crispin_client, db_session, log, folder_name,
uid_download_stack, len(local_uids),
len(remote_uids), shared_state['status_cb'],
shared_state['syncmanager_lock'],
gmail_download_and_commit_uids,
account.create_gmail_message, c)
# Complete X-GM-MSGID mapping is no longer needed after initial sync.
rm_cache(remote_g_metadata_cache_file(crispin_client.account_id,
folder_name))
new_uid_poller.kill()
