def condstore_imap_initial_sync(crispin_client, log, folder_name, shared_state,
local_uids, uid_download_stack, msg_create_fn):
with session_scope(ignore_soft_deletes=False) as db_session:
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
if saved_folder_info is None:
assert (crispin_client.selected_uidvalidity is not None
and crispin_client.selected_highestmodseq is not None)
account.update_folder_info(crispin_client.account_id, db_session,
folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
check_flags(crispin_client, db_session, log, folder_name, local_uids,
shared_state['syncmanager_lock'])
return imap_initial_sync(crispin_client,
log,
folder_name,
shared_state,
local_uids,
uid_download_stack,
msg_create_fn,
spawn_flags_refresh_poller=False)
def get_g_metadata(crispin_client, db_session, log, folder_name, uids,
syncmanager_lock):
assert folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
account_id = crispin_client.account_id
remote_g_metadata = None
update_uid_count = 0
saved_folder_info = account.get_folder_info(account_id,
db_session, folder_name)
if saved_folder_info is not None:
# If there's no cached validity we probably haven't run before.
sync_type = 'resumed'
remote_g_metadata, update_uid_count = retrieve_saved_g_metadata(
crispin_client, db_session, log, folder_name, uids,
saved_folder_info, syncmanager_lock)
if remote_g_metadata is None:
sync_type = 'new'
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(remote_g_metadata_cache_file(account_id, folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
account.update_folder_info(account_id, db_session, folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata, (sync_type, update_uid_count)
def get_g_metadata(crispin_client, log, folder_name, uids, syncmanager_lock):
assert folder_name == crispin_client.selected_folder_name, \
"crispin selected folder isn't as expected"
account_id = crispin_client.account_id
remote_g_metadata = None
update_uid_count = 0
with session_scope(ignore_soft_deletes=False) as db_session:
saved_folder_info = account.get_folder_info(
account_id, db_session, folder_name)
saved_highestmodseq = or_none(saved_folder_info, lambda i:
i.highestmodseq)
if saved_highestmodseq is not None:
# If there's no cached validity we probably haven't run before.
remote_g_metadata, update_uid_count = retrieve_saved_g_metadata(
crispin_client, log, folder_name, uids,
saved_highestmodseq, syncmanager_lock)
if remote_g_metadata is None:
remote_g_metadata = crispin_client.g_metadata(
crispin_client.all_uids())
set_cache(remote_g_metadata_cache_file(account_id, folder_name),
remote_g_metadata)
# Save highestmodseq that corresponds to the saved g_metadata.
with session_scope(ignore_soft_deletes=False) as db_session:
account.update_folder_info(account_id, db_session, folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
db_session.commit()
return remote_g_metadata, update_uid_count
def condstore_base_poll(crispin_client, db_session, log, folder_name,
shared_state, highestmodseq_fn):
""" Base polling logic for IMAP servers which support CONDSTORE and IDLE.
The CONDSTORE / HIGHESTMODSEQ mechanism is used to detect new and changed
messages that need syncing.
"""
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
# Start a session since we're going to IDLE below anyway...
# This also resets the folder name cache, which we want in order to
# detect folder/label additions and deletions.
status = crispin_client.select_folder(
folder_name, uidvalidity_cb(db_session, crispin_client.account_id))
log.debug("POLL current modseq: {} | saved modseq: {}".format(
status['HIGHESTMODSEQ'], saved_folder_info.highestmodseq))
if status['HIGHESTMODSEQ'] > saved_folder_info.highestmodseq:
acc = db_session.query(ImapAccount).get(crispin_client.account_id)
save_folder_names(log, acc, crispin_client.folder_names(), db_session)
highestmodseq_update(crispin_client, db_session, log, folder_name,
saved_folder_info.highestmodseq, highestmodseq_fn,
shared_state['syncmanager_lock'])
# We really only want to idle on a folder for new messages. Idling on
# `All Mail` won't tell us when messages are archived from the Inbox
if folder_name.lower() in IDLE_FOLDERS:
status = crispin_client.select_folder(
folder_name, uidvalidity_cb(db_session, crispin_client.account_id))
idle_frequency = 1800 # 30min
log.info("Idling on {0} with {1} timeout".format(
folder_name, idle_frequency))
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=idle_frequency)
# If we want to do something with the response, but lousy
# because it uses sequence IDs instead of UIDs
# resp = c.idle_check(timeout=shared_state['poll_frequency'])
# r = dict( EXISTS=[], EXPUNGE=[])
# for msg_uid, cmd in resp:
# r[cmd].append(msg_uid)
# print r
crispin_client.conn.idle_done()
log.info("IDLE triggered poll or timeout reached on {0}"
.format(folder_name))
else:
log.info("Sleeping on {0} for {1} seconds".format(
folder_name, shared_state['poll_frequency']))
sleep(shared_state['poll_frequency'])
return 'poll'
def check_flags(crispin_client, db_session, log, folder_name, local_uids,
syncmanager_lock):
""" Update message flags if folder has changed on the remote.
If we have saved validity info for this folder, make sure the folder hasn't
changed since we saved it. Otherwise we need to query for flag changes too.
"""
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
if saved_folder_info is not None:
last_highestmodseq = saved_folder_info.highestmodseq
if last_highestmodseq > crispin_client.selected_highestmodseq:
uids = crispin_client.new_and_updated_uids(last_highestmodseq)
if uids:
_, updated = new_or_updated(uids, local_uids)
update_metadata(crispin_client, db_session, log, folder_name,
updated, syncmanager_lock)
def check_flags(crispin_client, db_session, log, folder_name, local_uids,
syncmanager_lock):
""" Update message flags if folder has changed on the remote.
If we have saved validity info for this folder, make sure the folder hasn't
changed since we saved it. Otherwise we need to query for flag changes too.
"""
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
if saved_folder_info is not None:
last_highestmodseq = saved_folder_info.highestmodseq
if last_highestmodseq > crispin_client.selected_highestmodseq:
uids = crispin_client.new_and_updated_uids(last_highestmodseq)
if uids:
_, updated = new_or_updated(uids, local_uids)
update_metadata(crispin_client, db_session, log, folder_name,
updated, syncmanager_lock)
def fn(folder_name, select_info):
assert folder_name is not None and select_info is not None, \
"must start IMAP session before verifying UIDVALIDITY"
saved_folder_info = account.get_folder_info(account_id, db_session,
folder_name)
selected_uidvalidity = select_info['UIDVALIDITY']
if saved_folder_info:
is_valid = account.uidvalidity_valid(account_id, db_session,
selected_uidvalidity,
folder_name,
saved_folder_info.uidvalidity)
if not is_valid:
raise UidInvalid(
'folder: {}, remote uidvalidity: {}, '
'cached uidvalidity: {}'.format(
folder_name, selected_uidvalidity,
saved_folder_info.uidvalidity))
return select_info
def fn(folder_name, select_info):
assert folder_name is not None and select_info is not None, \
"must start IMAP session before verifying UIDVALIDITY"
with session_scope(ignore_soft_deletes=False) as db_session:
saved_folder_info = account.get_folder_info(
account_id, db_session, folder_name)
saved_uidvalidity = or_none(saved_folder_info,
lambda i: i.uidvalidity)
selected_uidvalidity = select_info['UIDVALIDITY']
if saved_folder_info:
is_valid = account.uidvalidity_valid(account_id,
selected_uidvalidity,
folder_name,
saved_uidvalidity)
if not is_valid:
raise UidInvalid('folder: {}, remote uidvalidity: {}, '
'cached uidvalidity: {}'.format(
folder_name, selected_uidvalidity,
saved_uidvalidity))
return select_info
def condstore_imap_initial_sync(crispin_client, log, folder_name,
shared_state, local_uids, uid_download_stack,
msg_create_fn):
with session_scope(ignore_soft_deletes=False) as db_session:
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
if saved_folder_info is None:
assert (crispin_client.selected_uidvalidity is not None and
crispin_client.selected_highestmodseq is not None)
account.update_folder_info(crispin_client.account_id, db_session,
folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
check_flags(crispin_client, db_session, log, folder_name, local_uids,
shared_state['syncmanager_lock'])
return imap_initial_sync(crispin_client, log, folder_name,
shared_state, local_uids, uid_download_stack,
msg_create_fn, spawn_flags_refresh_poller=False)
def fn(folder_name, select_info):
assert folder_name is not None and select_info is not None, \
"must start IMAP session before verifying UIDVALIDITY"
with session_scope(ignore_soft_deletes=False) as db_session:
saved_folder_info = account.get_folder_info(account_id, db_session,
folder_name)
saved_uidvalidity = or_none(saved_folder_info, lambda i:
i.uidvalidity)
selected_uidvalidity = select_info['UIDVALIDITY']
if saved_folder_info:
is_valid = account.uidvalidity_valid(account_id,
selected_uidvalidity,
folder_name,
saved_uidvalidity)
if not is_valid:
raise UidInvalid(
'folder: {}, remote uidvalidity: {}, '
'cached uidvalidity: {}'.format(
folder_name, selected_uidvalidity, saved_uidvalidity))
return select_info
def condstore_base_poll(crispin_client, log, folder_name, shared_state,
highestmodseq_fn):
""" Base polling logic for IMAP servers which support CONDSTORE and IDLE.
The CONDSTORE / HIGHESTMODSEQ mechanism is used to detect new and changed
messages that need syncing.
"""
log.bind(state='poll')
with session_scope(ignore_soft_deletes=False) as db_session:
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
saved_highestmodseq = saved_folder_info.highestmodseq
# Start a session since we're going to IDLE below anyway...
# This also resets the folder name cache, which we want in order to
# detect folder/label additions and deletions.
status = crispin_client.select_folder(
folder_name, uidvalidity_cb(crispin_client.account_id))
log.debug(current_modseq=status['HIGHESTMODSEQ'],
saved_modseq=saved_highestmodseq)
if status['HIGHESTMODSEQ'] > saved_highestmodseq:
with session_scope(ignore_soft_deletes=False) as db_session:
acc = db_session.query(ImapAccount).get(crispin_client.account_id)
save_folder_names(log, acc, crispin_client.folder_names(),
db_session)
highestmodseq_update(crispin_client, log, folder_name,
saved_highestmodseq, highestmodseq_fn,
shared_state['syncmanager_lock'])
# We really only want to idle on a folder for new messages. Idling on
# `All Mail` won't tell us when messages are archived from the Inbox
if folder_name.lower() in IDLE_FOLDERS:
status = crispin_client.select_folder(
folder_name, uidvalidity_cb(crispin_client.account_id))
idle_frequency = 1800 # 30min
log.info('idling', timeout=idle_frequency)
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=idle_frequency)
# If we want to do something with the response, but lousy
# because it uses sequence IDs instead of UIDs
# resp = c.idle_check(timeout=shared_state['poll_frequency'])
# r = dict( EXISTS=[], EXPUNGE=[])
# for msg_uid, cmd in resp:
# r[cmd].append(msg_uid)
# print r
crispin_client.conn.idle_done()
log.info('IDLE triggered poll')
else:
log.info('IDLE sleeping', seconds=shared_state['poll_frequency'])
sleep(shared_state['poll_frequency'])
return 'poll'
