def highestmodseq_update(crispin_client, log, folder_name, last_highestmodseq,
highestmodseq_fn, syncmanager_lock):
account_id = crispin_client.account_id
new_highestmodseq = crispin_client.selected_highestmodseq
new_uidvalidity = crispin_client.selected_uidvalidity
log.info('starting highestmodseq update',
current_highestmodseq=new_highestmodseq)
changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
remote_uids = crispin_client.all_uids()
local_uids = None
if changed_uids:
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = account.all_uids(account_id, db_session, folder_name)
new, updated = new_or_updated(changed_uids, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
local_uids += new
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
deleted_uids = remove_deleted_uids(account_id, db_session, log,
folder_name, local_uids,
remote_uids)
local_uids = set(local_uids) - deleted_uids
update_metadata(crispin_client, log, folder_name, updated,
syncmanager_lock)
with session_scope(ignore_soft_deletes=False) as db_session:
update_uid_counts(db_session, log, account_id, folder_name,
remote_uid_count=len(remote_uids),
download_uid_count=len(new),
update_uid_count=len(updated),
delete_uid_count=len(deleted_uids))
highestmodseq_fn(crispin_client, log, folder_name, new,
updated, syncmanager_lock)
else:
log.info("No new or updated messages")
with session_scope(ignore_soft_deletes=False) as db_session:
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
if local_uids is None:
local_uids = account.all_uids(
account_id, db_session, folder_name)
deleted_uids = remove_deleted_uids(crispin_client.account_id,
db_session, log, folder_name,
local_uids, remote_uids)
update_uid_counts(db_session, log, account_id, folder_name,
remote_uid_count=len(remote_uids),
delete_uid_count=len(deleted_uids))
account.update_folder_info(account_id, db_session, folder_name,
new_uidvalidity, new_highestmodseq)
db_session.commit()
def base_initial_sync(crispin_client, log, folder_name, shared_state,
initial_sync_fn, msg_create_fn):
""" Downloads entire messages.
This function may be retried as many times as you like; it will pick up
where it left off, delete removed messages if things disappear between
restarts, and only complete once we have all the UIDs in the given folder
locally.
This function also starts up a secondary greenlet that checks for new
messages periodically, to deal with the case of very large folders---it's
a bad experience for the user to keep receiving old mail but not receive
new mail! We use a LIFO queue to make sure we're downloading newest mail
first.
"""
log.info('starting initial sync')
uid_download_stack = LifoQueue()
crispin_client.select_folder(folder_name,
uidvalidity_cb(crispin_client.account_id))
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = account.all_uids(crispin_client.account_id, db_session,
folder_name)
initial_sync_fn(crispin_client, log, folder_name, shared_state, local_uids,
uid_download_stack, msg_create_fn)
return 'poll'
def base_poll(crispin_client, db_session, log, folder_name, shared_state,
download_fn, msg_create_fn):
""" Base polling logic for non-CONDSTORE IMAP servers.
Local/remote UID comparison is used to detect new and deleted messages.
Currently does not support synchronizing flag changes.
"""
account_id = crispin_client.account_id
status = crispin_client.select_folder(
folder_name,
uidvalidity_cb(db_session, account_id))
log.debug("POLL current UIDNEXT: {}".format(status['UIDNEXT']))
remote_uids = set(crispin_client.all_uids())
local_uids = set(account.all_uids(
account_id, db_session, folder_name))
deleted_uids = remove_deleted_uids(
account_id, db_session, log, folder_name, local_uids,
remote_uids)
local_uids -= deleted_uids
log.info("Removed {} deleted UIDs from {}".format(
len(deleted_uids), folder_name))
to_download = remote_uids - local_uids
log.info("UIDs to download: {}".format(to_download))
if to_download:
download_fn(crispin_client, db_session, log, folder_name,
to_download, local_uids, shared_state['syncmanager_lock'],
msg_create_fn)
sleep(shared_state['poll_frequency'])
return 'poll'
def base_initial_sync(crispin_client, log, folder_name, shared_state,
initial_sync_fn, msg_create_fn):
""" Downloads entire messages.
This function may be retried as many times as you like; it will pick up
where it left off, delete removed messages if things disappear between
restarts, and only complete once we have all the UIDs in the given folder
locally.
This function also starts up a secondary greenlet that checks for new
messages periodically, to deal with the case of very large folders---it's
a bad experience for the user to keep receiving old mail but not receive
new mail! We use a LIFO queue to make sure we're downloading newest mail
first.
"""
log.info('starting initial sync')
uid_download_stack = LifoQueue()
crispin_client.select_folder(folder_name,
uidvalidity_cb(crispin_client.account_id))
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = account.all_uids(crispin_client.account_id,
db_session, folder_name)
initial_sync_fn(crispin_client, log, folder_name, shared_state,
local_uids, uid_download_stack, msg_create_fn)
return 'poll'
def check_new_uids(account_id, folder_name, log, uid_download_stack,
poll_frequency, syncmanager_lock):
""" Check for new UIDs and add them to the download stack.
We do this by comparing local UID lists to remote UID lists, maintaining
the invariant that (stack uids)+(local uids) == (remote uids).
We also remove local messages that have disappeared from the remote, since
it's totally probable that users will be archiving mail as the initial
sync goes on.
We grab a new IMAP connection from the pool for this to isolate its
actions from whatever the main greenlet may be doing.
Runs until killed. (Intended to be run in a greenlet.)
"""
log.info("starting new UID-check poller")
with _pool(account_id).get() as crispin_client:
crispin_client.select_folder(
folder_name, uidvalidity_cb(crispin_client.account_id))
while True:
remote_uids = set(crispin_client.all_uids())
# We lock this section to make sure no messages are being
# created while we make sure the queue is in a good state.
with syncmanager_lock:
log.debug("check_new_uids acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(account.all_uids(account_id, db_session,
folder_name))
stack_uids = set(uid_download_stack.queue)
local_with_pending_uids = local_uids | stack_uids
deleted_uids = remove_deleted_uids(
account_id, db_session, log, folder_name, local_uids,
remote_uids)
log.info('remoted deleted uids', count=len(deleted_uids))
# filter out messages that have disappeared on the
# remote side
new_uid_download_stack = {u for u in
uid_download_stack.queue if u in
remote_uids}
# add in any new uids from the remote
for uid in remote_uids:
if uid not in local_with_pending_uids:
log.debug("adding new message {} to download queue"
.format(uid))
new_uid_download_stack.add(uid)
uid_download_stack.queue = sorted(new_uid_download_stack,
key=int)
update_uid_counts(
db_session, log, crispin_client.account_id,
folder_name, remote_uid_count=len(remote_uids),
download_uid_count=uid_download_stack.qsize(),
delete_uid_count=len(deleted_uids))
sleep(poll_frequency)
def base_poll(crispin_client, log, folder_name, shared_state, download_fn,
msg_create_fn, update_fn):
""" Base polling logic for non-CONDSTORE IMAP servers.
Local/remote UID comparison is used to detect new and deleted messages.
Currently does not support synchronizing flag changes.
"""
account_id = crispin_client.account_id
with session_scope(ignore_soft_deletes=False) as db_session:
status = crispin_client.select_folder(folder_name,
uidvalidity_cb(account_id))
log.debug("POLL current UIDNEXT: {}".format(status['UIDNEXT']))
remote_uids = set(crispin_client.all_uids())
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(account.all_uids(account_id, db_session, folder_name))
deleted_uids = remove_deleted_uids(account_id, db_session, log,
folder_name, local_uids,
remote_uids)
local_uids -= deleted_uids
log.info("Removed {} deleted UIDs from {}".format(
len(deleted_uids), folder_name))
to_download = remote_uids - local_uids
update_uid_counts(db_session,
log,
account_id,
folder_name,
remote_uid_count=len(remote_uids),
download_uid_count=len(to_download),
delete_uid_count=len(deleted_uids))
log.info("UIDs to download: {}".format(to_download))
if to_download:
download_fn(crispin_client, log, folder_name, to_download, local_uids,
shared_state['syncmanager_lock'], msg_create_fn)
flags_max_nr = abs(shared_state['refresh_flags_max'])
to_refresh = sorted(remote_uids - to_download)[-flags_max_nr:]
log.info('UIDs to refresh: ', uids=to_refresh)
if to_refresh:
update_fn(crispin_client, log, folder_name, to_refresh,
shared_state['syncmanager_lock'])
sleep(shared_state['poll_frequency'])
return 'poll'
def base_poll(crispin_client, log, folder_name, shared_state, download_fn,
msg_create_fn, update_fn):
""" Base polling logic for non-CONDSTORE IMAP servers.
Local/remote UID comparison is used to detect new and deleted messages.
Currently does not support synchronizing flag changes.
"""
account_id = crispin_client.account_id
with session_scope(ignore_soft_deletes=False) as db_session:
status = crispin_client.select_folder(
folder_name, uidvalidity_cb(account_id))
log.debug("POLL current UIDNEXT: {}".format(status['UIDNEXT']))
remote_uids = set(crispin_client.all_uids())
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(account.all_uids(
account_id, db_session, folder_name))
deleted_uids = remove_deleted_uids(
account_id, db_session, log, folder_name, local_uids,
remote_uids)
local_uids -= deleted_uids
log.info("Removed {} deleted UIDs from {}".format(
len(deleted_uids), folder_name))
to_download = remote_uids - local_uids
update_uid_counts(db_session, log, account_id, folder_name,
remote_uid_count=len(remote_uids),
download_uid_count=len(to_download),
delete_uid_count=len(deleted_uids))
log.info("UIDs to download: {}".format(to_download))
if to_download:
download_fn(crispin_client, log, folder_name, to_download, local_uids,
shared_state['syncmanager_lock'], msg_create_fn)
flags_max_nr = abs(shared_state['refresh_flags_max'])
to_refresh = sorted(remote_uids - to_download)[-flags_max_nr:]
log.info('UIDs to refresh: ', uids=to_refresh)
if to_refresh:
update_fn(crispin_client, log, folder_name, to_refresh,
shared_state['syncmanager_lock'])
sleep(shared_state['poll_frequency'])
return 'poll'
def imap_check_flags(account_id, folder_name, log, poll_frequency,
syncmanager_lock, refresh_flags_max):
"""
Periodically update message flags for those servers
who don't support CONDSTORE.
Runs until killed. (Intended to be run in a greenlet)
Parameters
----------
account_id : String
folder_name : String
log : Logger
poll_frequency : Integer
Number of seconds to wait between polls.
syncmanager_lock : Locking Context Manager
refresh_flags_max : Integer
Maximum number of messages to check FLAGS of.
"""
log.info("Spinning up new flags-refresher for ", folder_name=folder_name)
with connection_pool(account_id).get() as crispin_client:
with session_scope(ignore_soft_deletes=False) as db_session:
crispin_client.select_folder(folder_name,
uidvalidity_cb(
db_session,
crispin_client.account_id))
while True:
remote_uids = set(crispin_client.all_uids())
local_uids = set(account.all_uids(account_id, db_session,
folder_name))
to_refresh = sorted(remote_uids & local_uids)[-refresh_flags_max:]
update_metadata(crispin_client,
db_session,
log,
folder_name,
to_refresh,
syncmanager_lock)
update_uid_counts(db_session,
log,
crispin_client.account_id,
folder_name,
update_uid_count=len(to_refresh))
sleep(poll_frequency)
def highestmodseq_update(crispin_client, db_session, log, folder_name,
last_highestmodseq, highestmodseq_fn,
syncmanager_lock):
account_id = crispin_client.account_id
new_highestmodseq = crispin_client.selected_highestmodseq
new_uidvalidity = crispin_client.selected_uidvalidity
log.info("Starting highestmodseq update on {} (current HIGHESTMODSEQ: {})"
.format(folder_name, new_highestmodseq))
local_uids = account.all_uids(account_id, db_session, folder_name)
changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
remote_uids = crispin_client.all_uids()
if changed_uids:
new, updated = new_or_updated(changed_uids, local_uids)
log.info("{0} new and {1} updated UIDs".format(len(new), len(updated)))
local_uids += new
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
deleted_uids = remove_deleted_uids(account_id, db_session, log,
folder_name, local_uids,
remote_uids)
local_uids = set(local_uids) - deleted_uids
update_metadata(crispin_client, db_session, log, folder_name,
updated, syncmanager_lock)
update_uid_counts(db_session, log, account_id, folder_name,
remote_uid_count=len(remote_uids),
download_uid_count=len(new),
update_uid_count=len(updated),
delete_uid_count=len(deleted_uids))
highestmodseq_fn(crispin_client, db_session, log, folder_name,
changed_uids, local_uids, syncmanager_lock)
else:
log.info("No new or updated messages")
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
remove_deleted_uids(crispin_client.account_id, db_session, log,
folder_name, local_uids, remote_uids)
account.update_folder_info(account_id, db_session, folder_name,
new_uidvalidity, new_highestmodseq)
db_session.commit()
def imap_check_flags(account_id, folder_name, log, poll_frequency,
syncmanager_lock, refresh_flags_max):
"""
Periodically update message flags for those servers
who don't support CONDSTORE.
Runs until killed. (Intended to be run in a greenlet)
Parameters
----------
account_id : String
folder_name : String
log : Logger
poll_frequency : Integer
Number of seconds to wait between polls.
syncmanager_lock : Locking Context Manager
refresh_flags_max : Integer
Maximum number of messages to check FLAGS of.
"""
log.info("Spinning up new flags-refresher for ", folder_name=folder_name)
with _pool(account_id).get() as crispin_client:
with session_scope(ignore_soft_deletes=False) as db_session:
crispin_client.select_folder(
folder_name, uidvalidity_cb(crispin_client.account_id))
while True:
remote_uids = set(crispin_client.all_uids())
local_uids = set(
account.all_uids(account_id, db_session, folder_name))
to_refresh = sorted(remote_uids & local_uids)[-refresh_flags_max:]
update_metadata(crispin_client, log, folder_name, to_refresh,
syncmanager_lock)
with session_scope(ignore_soft_deletes=True) as db_session:
update_uid_counts(db_session,
log,
crispin_client.account_id,
folder_name,
update_uid_count=len(to_refresh))
sleep(poll_frequency)
def check_new_uids(account_id, folder_name, log, uid_download_stack,
poll_frequency, syncmanager_lock):
""" Check for new UIDs and add them to the download stack.
We do this by comparing local UID lists to remote UID lists, maintaining
the invariant that (stack uids)+(local uids) == (remote uids).
We also remove local messages that have disappeared from the remote, since
it's totally probable that users will be archiving mail as the initial
sync goes on.
We grab a new IMAP connection from the pool for this to isolate its
actions from whatever the main greenlet may be doing.
Runs until killed. (Intended to be run in a greenlet.)
"""
log.info("starting new UID-check poller")
with _pool(account_id).get() as crispin_client:
crispin_client.select_folder(folder_name,
uidvalidity_cb(crispin_client.account_id))
while True:
remote_uids = set(crispin_client.all_uids())
# We lock this section to make sure no messages are being
# created while we make sure the queue is in a good state.
with syncmanager_lock:
log.debug("check_new_uids acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(
account.all_uids(account_id, db_session, folder_name))
stack_uids = set(uid_download_stack.queue)
local_with_pending_uids = local_uids | stack_uids
deleted_uids = remove_deleted_uids(account_id, db_session,
log, folder_name,
local_uids, remote_uids)
log.info('remoted deleted uids', count=len(deleted_uids))
# filter out messages that have disappeared on the
# remote side
new_uid_download_stack = {
u
for u in uid_download_stack.queue if u in remote_uids
}
# add in any new uids from the remote
for uid in remote_uids:
if uid not in local_with_pending_uids:
log.debug(
"adding new message {} to download queue".
format(uid))
new_uid_download_stack.add(uid)
uid_download_stack.queue = sorted(new_uid_download_stack,
key=int)
update_uid_counts(
db_session,
log,
crispin_client.account_id,
folder_name,
remote_uid_count=len(remote_uids),
download_uid_count=uid_download_stack.qsize(),
delete_uid_count=len(deleted_uids))
sleep(poll_frequency)
def highestmodseq_update(crispin_client, log, folder_name, last_highestmodseq,
highestmodseq_fn, syncmanager_lock):
account_id = crispin_client.account_id
new_highestmodseq = crispin_client.selected_highestmodseq
new_uidvalidity = crispin_client.selected_uidvalidity
log.info('starting highestmodseq update',
current_highestmodseq=new_highestmodseq)
changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
remote_uids = crispin_client.all_uids()
local_uids = None
if changed_uids:
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = account.all_uids(account_id, db_session, folder_name)
new, updated = new_or_updated(changed_uids, local_uids)
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
local_uids += new
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
deleted_uids = remove_deleted_uids(account_id, db_session, log,
folder_name, local_uids,
remote_uids)
local_uids = set(local_uids) - deleted_uids
update_metadata(crispin_client, log, folder_name, updated,
syncmanager_lock)
with session_scope(ignore_soft_deletes=False) as db_session:
update_uid_counts(db_session,
log,
account_id,
folder_name,
remote_uid_count=len(remote_uids),
download_uid_count=len(new),
update_uid_count=len(updated),
delete_uid_count=len(deleted_uids))
highestmodseq_fn(crispin_client, log, folder_name, new, updated,
syncmanager_lock)
else:
log.info("No new or updated messages")
with session_scope(ignore_soft_deletes=False) as db_session:
with syncmanager_lock:
log.debug("highestmodseq_update acquired syncmanager_lock")
if local_uids is None:
local_uids = account.all_uids(account_id, db_session,
folder_name)
deleted_uids = remove_deleted_uids(crispin_client.account_id,
db_session, log, folder_name,
local_uids, remote_uids)
update_uid_counts(db_session,
log,
account_id,
folder_name,
remote_uid_count=len(remote_uids),
delete_uid_count=len(deleted_uids))
account.update_folder_info(account_id, db_session, folder_name,
new_uidvalidity, new_highestmodseq)
db_session.commit()
def check_new_g_thrids(account_id, provider, folder_name, log,
message_download_stack, poll_frequency,
syncmanager_lock):
"""
Check for new X-GM-THRIDs and add them to the download stack.
We do this by comparing local UID lists to remote UID lists, maintaining
the invariant that (stack uids)+(local uids) == (remote uids).
We also remove local messages that have disappeared from the remote, since
it's totally probable that users will be archiving mail as the initial
sync goes on.
We grab a new IMAP connection from the pool for this to isolate its
actions from whatever the main greenlet may be doing.
Runs until killed. (Intended to be run in a greenlet.)
"""
with connection_pool(account_id).get() as crispin_client:
with session_scope(ignore_soft_deletes=False) as db_session:
crispin_client.select_folder(folder_name,
uidvalidity_cb(
db_session,
crispin_client.account_id))
while True:
log.info('Checking for new/deleted messages during initial sync.')
remote_uids = set(crispin_client.all_uids())
# We lock this section to make sure no messages are being modified
# in the database while we make sure the queue is in a good state.
with syncmanager_lock:
log.debug('check_new_g_thrids acquired syncmanager_lock')
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(account.all_uids(account_id, db_session,
folder_name))
stack_uids = {gm.uid for gm in
message_download_stack.queue}
local_with_pending_uids = local_uids | stack_uids
deleted_uids = remove_deleted_uids(
account_id, db_session, log, folder_name, local_uids,
remote_uids)
log.info(deleted_uid_count=len(deleted_uids))
# filter out messages that have disappeared on the remote side
new_message_download_stack = [gm for gm in
message_download_stack.queue
if gm.uid in remote_uids]
# add in any new uids from the remote
new_uids = [uid for uid in remote_uids if uid not in
local_with_pending_uids]
flags = crispin_client.flags(new_uids)
g_metadata = crispin_client.g_metadata(new_uids)
log.info('adding new messages to download queue',
count=min(len(flags), len(g_metadata)))
for new_uid in new_uids:
# could have disappeared from the folder in the meantime
if new_uid in flags and new_uid in g_metadata:
new_message_download_stack.append(
GMessage(new_uid, g_metadata[new_uid],
flags[new_uid].flags,
flags[new_uid].labels))
message_download_stack.queue = sorted(
new_message_download_stack, key=lambda m: m.uid)
log.info('idling', timeout=poll_frequency)
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=poll_frequency)
crispin_client.conn.idle_done()
log.info('IDLE detected changes or timeout reached')
def check_new_g_thrids(account_id, provider_name, folder_name, log,
message_download_stack, poll_frequency,
syncmanager_lock):
"""
Check for new X-GM-THRIDs and add them to the download stack.
We do this by comparing local UID lists to remote UID lists, maintaining
the invariant that (stack uids)+(local uids) == (remote uids).
We also remove local messages that have disappeared from the remote, since
it's totally probable that users will be archiving mail as the initial
sync goes on.
We grab a new IMAP connection from the pool for this to isolate its
actions from whatever the main greenlet may be doing.
Runs until killed. (Intended to be run in a greenlet.)
"""
with connection_pool(account_id).get() as crispin_client:
crispin_client.select_folder(folder_name,
uidvalidity_cb(crispin_client.account_id))
while True:
log.info('Checking for new/deleted messages during initial sync.')
remote_uids = set(crispin_client.all_uids())
# We lock this section to make sure no messages are being modified
# in the database while we make sure the queue is in a good state.
with syncmanager_lock:
log.debug('check_new_g_thrids acquired syncmanager_lock')
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(account.all_uids(account_id, db_session,
folder_name))
stack_uids = {gm.uid for gm in
message_download_stack.queue}
local_with_pending_uids = local_uids | stack_uids
deleted_uids = remove_deleted_uids(
account_id, db_session, log, folder_name, local_uids,
remote_uids)
log.info(deleted_uid_count=len(deleted_uids))
# filter out messages that have disappeared on the remote side
new_message_download_stack = [gm for gm in
message_download_stack.queue
if gm.uid in remote_uids]
# add in any new uids from the remote
new_uids = [uid for uid in remote_uids if uid not in
local_with_pending_uids]
flags = crispin_client.flags(new_uids)
g_metadata = crispin_client.g_metadata(new_uids)
log.info('adding new messages to download queue',
count=min(len(flags), len(g_metadata)))
for new_uid in new_uids:
# could have disappeared from the folder in the meantime
if new_uid in flags and new_uid in g_metadata:
new_message_download_stack.append(
GMessage(new_uid, g_metadata[new_uid],
flags[new_uid].flags,
flags[new_uid].labels))
message_download_stack.queue = sorted(
new_message_download_stack, key=lambda m: m.uid)
with session_scope(ignore_soft_deletes=False) as db_session:
update_uid_counts(
db_session, log, crispin_client.account_id,
folder_name, remote_uid_count=len(remote_uids),
download_uid_count=message_download_stack.qsize(),
delete_uid_count=len(deleted_uids))
log.info('idling', timeout=poll_frequency)
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=poll_frequency)
crispin_client.conn.idle_done()
log.info('IDLE detected changes or timeout reached')
