def test_handle_trailing_whitespace(db, default_account, folder_name_mapping):
folder_name_mapping["extra"] = ["label", "label "]
log = get_logger()
save_folder_names(log, default_account.id, folder_name_mapping, db.session)
# Would raise if tag for label was not committed.
db.session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="label").one()
def test_sync_folder_deletes(db, default_account, folder_name_mapping):
"""Test that folder deletions properly cascade to deletions of
ImapFolderSyncStatus and ImapFolderInfo.
"""
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, default_account.id, folder_name_mapping, db_session)
folders = db_session.query(Folder).filter_by(account_id=default_account.id)
for folder in folders:
add_imap_status_info_rows(folder.id, default_account.id, db_session)
db_session.commit()
assert db_session.query(ImapFolderInfo).filter_by(account_id=default_account.id).count() == 7
assert db_session.query(ImapFolderSyncStatus).filter_by(account_id=default_account.id).count() == 7
folder_name_mapping["extra"] = ["Jobslist"]
save_folder_names(log, default_account.id, folder_name_mapping, db_session)
saved_folder_names = {
name for name, in db_session.query(Folder.name).filter(Folder.account_id == default_account.id)
}
assert saved_folder_names == {
"Inbox",
"[Gmail]/Spam",
"[Gmail]/All Mail",
"[Gmail]/Sent Mail",
"[Gmail]/Drafts",
"Jobslist",
}
assert db_session.query(ImapFolderInfo).filter_by(account_id=default_account.id).count() == 6
assert db_session.query(ImapFolderSyncStatus).filter_by(account_id=default_account.id).count() == 6
def prepare_sync(self):
"""Ensures that canonical tags are created for the account, and gets
and save Folder objects for folders on the IMAP backend. Returns a list
of tuples (folder_name, folder_id) for each folder we want to sync (in
order)."""
with mailsync_session_scope() as db_session:
with connection_pool(self.account_id).get() as crispin_client:
# the folders we should be syncing
sync_folders = crispin_client.sync_folders()
# get a fresh list of the folder names from the remote
remote_folders = crispin_client.folder_names(force_resync=True)
save_folder_names(log, self.account_id,
remote_folders, db_session)
sync_folder_names_ids = []
for folder_name in sync_folders:
try:
id_, = db_session.query(Folder.id). \
filter(Folder.name == folder_name,
Folder.account_id == self.account_id).one()
sync_folder_names_ids.append((folder_name, id_))
except NoResultFound:
log.error("Missing Folder object when starting sync",
folder_name=folder_name)
raise MailsyncError("Missing Folder '{}' on account {}"
.format(folder_name, self.account_id))
return sync_folder_names_ids
def check_uid_changes(self, crispin_client, download_stack,
async_download):
crispin_client.select_folder(self.folder_name, uidvalidity_cb)
new_highestmodseq = crispin_client.selected_highestmodseq
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(
self.account_id, db_session, self.folder_name)
# Ensure that we have an initial highestmodseq value stored before
# we begin polling for changes.
if saved_folder_info is None or \
saved_folder_info.highestmodseq is None:
assert (crispin_client.selected_uidvalidity is not None
and crispin_client.selected_highestmodseq is
not None)
saved_folder_info = common.update_folder_info(
crispin_client.account_id, db_session,
self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
saved_highestmodseq = saved_folder_info.highestmodseq
if new_highestmodseq == saved_highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < saved_highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning('got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=saved_highestmodseq)
return
save_folder_names(log, self.account_id,
crispin_client.folder_names(), db_session)
# Highestmodseq has changed, update accordingly.
new_uidvalidity = crispin_client.selected_uidvalidity
changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
remote_uids = crispin_client.all_uids()
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
stack_uids = {uid for uid, _ in download_stack}
local_with_pending_uids = local_uids | stack_uids
new, updated = new_or_updated(changed_uids, local_with_pending_uids)
if changed_uids:
log.info("Changed UIDs", message="new: {} updated: {}"
.format(len(new), len(updated)),
new_uid_count=len(new), updated_uid_count=len(updated))
self.update_metadata(crispin_client, updated)
self.highestmodseq_callback(crispin_client, new, updated,
download_stack, async_download)
with mailsync_session_scope() as db_session:
with self.syncmanager_lock:
self.remove_deleted_uids(db_session, local_uids, remote_uids)
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def prepare_sync(self):
"""Ensures that canonical tags are created for the account, and gets
and save Folder objects for folders on the IMAP backend. Returns a list
of tuples (folder_name, folder_id) for each folder we want to sync (in
order)."""
with mailsync_session_scope() as db_session:
account = db_session.query(ImapAccount).get(self.account_id)
Tag.create_canonical_tags(account.namespace, db_session)
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
save_folder_names(log, self.account_id,
crispin_client.folder_names(), db_session)
sync_folder_names_ids = []
for folder_name in sync_folders:
try:
id_, = db_session.query(Folder.id). \
filter(Folder.name == folder_name,
Folder.account_id == self.account_id).one()
sync_folder_names_ids.append((folder_name, id_))
except NoResultFound:
log.error("Missing Folder object when starting sync",
folder_name=folder_name)
raise MailsyncError("Missing Folder '{}' on account {}"
.format(folder_name, self.account_id))
return sync_folder_names_ids
def test_name_collision_folders(db, default_account, folder_name_mapping):
# test that when a user-created folder called 'spam' is created, we don't
# associate it with the canonical spam tag, but instead give it its own
# tag
folder_name_mapping["extra"] = ["spam"]
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, default_account.id, folder_name_mapping, db_session)
spam_tags = db_session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="spam")
# There should be one 'Gmail/Spam' canonical tag
assert spam_tags.count() == 1
assert spam_tags.first().public_id == "spam"
# and one 'imap/spam' non-canonical tag with public_id != 'spam'
spam_tags = db_session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="imap/spam")
assert spam_tags.count() == 1
assert spam_tags.first().public_id != "spam"
# test that when a folder called 'spam' is deleted, we don't delete
# the canonical 'spam' tag
folder_name_mapping["extra"] = []
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, default_account.id, folder_name_mapping, db_session)
spam_tags = db_session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="spam")
# The 'Gmail/Spam' canonical tag should still remain.
assert spam_tags.count() == 1
assert spam_tags.first().public_id == "spam"
# The 'imap/spam' non-canonical tag shouldn't
spam_tags = db_session.query(Tag).filter_by(namespace_id=default_account.namespace.id, name="imap/spam")
assert spam_tags.count() == 0
def test_handle_trailing_whitespace(db, default_account, folder_name_mapping):
folder_name_mapping['extra'] = ['label', 'label ']
log = get_logger()
save_folder_names(log, default_account.id, folder_name_mapping, db.session)
# Would raise if tag for label was not committed.
db.session.query(Tag).filter_by(namespace_id=default_account.namespace.id,
name='label').one()
def create_foldersyncstatuses(db, default_account):
# Create a bunch of folder sync statuses.
mapping = folder_name_mapping()
save_folder_names(log, default_account.id, mapping, db.session)
folders = db.session.query(Folder).filter_by(account_id=default_account.id)
for folder in folders:
add_imap_status_info_rows(folder.id, default_account.id, db.session)
db.session.commit()
def create_foldersyncstatuses(db, default_account):
# Create a bunch of folder sync statuses.
mapping = folder_name_mapping()
save_folder_names(log, default_account.id, mapping, db.session)
folders = db.session.query(Folder).filter_by(account_id=default_account.id)
for folder in folders:
add_imap_status_info_rows(folder.id, default_account.id, db.session)
db.session.commit()
def check_uid_changes(self, crispin_client, download_stack,
async_download):
crispin_client.select_folder(self.folder_name, uidvalidity_cb)
new_highestmodseq = crispin_client.selected_highestmodseq
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(self.account_id,
db_session,
self.folder_name)
# Ensure that we have an initial highestmodseq value stored before
# we begin polling for changes.
if saved_folder_info is None:
assert (crispin_client.selected_uidvalidity is not None
and crispin_client.selected_highestmodseq is not None)
saved_folder_info = common.update_folder_info(
crispin_client.account_id, db_session, self.folder_name,
crispin_client.selected_uidvalidity,
crispin_client.selected_highestmodseq)
saved_highestmodseq = saved_folder_info.highestmodseq
if new_highestmodseq == saved_highestmodseq:
# Don't need to do anything if the highestmodseq hasn't
# changed.
return
elif new_highestmodseq < saved_highestmodseq:
# This should really never happen, but if it does, handle it.
log.warning(
'got server highestmodseq less than saved '
'highestmodseq',
new_highestmodseq=new_highestmodseq,
saved_highestmodseq=saved_highestmodseq)
return
save_folder_names(log, self.account_id,
crispin_client.folder_names(), db_session)
# Highestmodseq has changed, update accordingly.
new_uidvalidity = crispin_client.selected_uidvalidity
changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
remote_uids = crispin_client.all_uids()
with self.syncmanager_lock:
with mailsync_session_scope() as db_session:
local_uids = common.all_uids(self.account_id, db_session,
self.folder_name)
self.remove_deleted_uids(db_session, local_uids, remote_uids)
stack_uids = {uid for uid, _ in download_stack}
local_with_pending_uids = local_uids | stack_uids
new, updated = new_or_updated(changed_uids,
local_with_pending_uids)
if changed_uids:
log.info(new_uid_count=len(new), updated_uid_count=len(updated))
self.update_metadata(crispin_client, updated)
self.highestmodseq_callback(crispin_client, new, updated,
download_stack, async_download)
with mailsync_session_scope() as db_session:
self.update_uid_counts(db_session,
remote_uid_count=len(remote_uids))
common.update_folder_info(self.account_id, db_session,
self.folder_name, new_uidvalidity,
new_highestmodseq)
db_session.commit()
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with mailsync_session_scope() as db_session:
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(log, account, crispin_client.folder_names(),
db_session)
Tag.create_canonical_tags(account.namespace, db_session)
folder_id_for = {
name: id_
for id_, name in db_session.query(Folder.id, Folder.name).
filter_by(account_id=self.account_id)
}
saved_states = {
name: state
for name, state in db_session.query(
Folder.name, ImapFolderSyncStatus.state).join(
ImapFolderSyncStatus.folder).filter(
ImapFolderSyncStatus.account_id == self.account_id)
}
for folder_name in sync_folders:
if folder_name not in folder_id_for:
log.error("Missing Folder object when starting sync",
folder_name=folder_name,
folder_id_for=folder_id_for)
raise MailsyncError("Missing Folder '{}' on account {}".format(
folder_name, self.account_id))
if saved_states.get(folder_name) != 'finish':
log.info('initializing folder sync')
# STOPSHIP(emfree): replace by appropriate base class.
thread = self.sync_engine_class(
self.account_id, folder_name, folder_id_for[folder_name],
self.email_address, self.provider_name,
self.poll_frequency, self.syncmanager_lock,
self.refresh_flags_max, self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread) or thread.ready():
log.info('folder sync finished/killed',
folder_name=thread.folder_name)
# NOTE: Greenlet is automatically removed from the group.
self.folder_monitors.join()
def condstore_base_poll(crispin_client, db_session, log, folder_name,
shared_state, highestmodseq_fn):
""" Base polling logic for IMAP servers which support CONDSTORE and IDLE.
The CONDSTORE / HIGHESTMODSEQ mechanism is used to detect new and changed
messages that need syncing.
"""
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
# Start a session since we're going to IDLE below anyway...
# This also resets the folder name cache, which we want in order to
# detect folder/label additions and deletions.
status = crispin_client.select_folder(
folder_name, uidvalidity_cb(db_session, crispin_client.account_id))
log.debug("POLL current modseq: {} | saved modseq: {}".format(
status['HIGHESTMODSEQ'], saved_folder_info.highestmodseq))
if status['HIGHESTMODSEQ'] > saved_folder_info.highestmodseq:
acc = db_session.query(ImapAccount).get(crispin_client.account_id)
save_folder_names(log, acc, crispin_client.folder_names(), db_session)
highestmodseq_update(crispin_client, db_session, log, folder_name,
saved_folder_info.highestmodseq, highestmodseq_fn,
shared_state['syncmanager_lock'])
# We really only want to idle on a folder for new messages. Idling on
# `All Mail` won't tell us when messages are archived from the Inbox
if folder_name.lower() in IDLE_FOLDERS:
status = crispin_client.select_folder(
folder_name, uidvalidity_cb(db_session, crispin_client.account_id))
idle_frequency = 1800 # 30min
log.info("Idling on {0} with {1} timeout".format(
folder_name, idle_frequency))
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=idle_frequency)
# If we want to do something with the response, but lousy
# because it uses sequence IDs instead of UIDs
# resp = c.idle_check(timeout=shared_state['poll_frequency'])
# r = dict( EXISTS=[], EXPUNGE=[])
# for msg_uid, cmd in resp:
# r[cmd].append(msg_uid)
# print r
crispin_client.conn.idle_done()
log.info("IDLE triggered poll or timeout reached on {0}"
.format(folder_name))
else:
log.info("Sleeping on {0} for {1} seconds".format(
folder_name, shared_state['poll_frequency']))
sleep(shared_state['poll_frequency'])
return 'poll'
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with mailsync_session_scope() as db_session:
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(log, account,
crispin_client.folder_names(), db_session)
Tag.create_canonical_tags(account.namespace, db_session)
folder_id_for = {name: id_ for id_, name in db_session.query(
Folder.id, Folder.name).filter_by(account_id=self.account_id)}
saved_states = {name: state for name, state in
db_session.query(Folder.name,
ImapFolderSyncStatus.state)
.join(ImapFolderSyncStatus.folder)
.filter(ImapFolderSyncStatus.account_id ==
self.account_id)}
for folder_name in sync_folders:
if folder_name not in folder_id_for:
log.error("Missing Folder object when starting sync",
folder_name=folder_name, folder_id_for=folder_id_for)
raise MailsyncError("Missing Folder '{}' on account {}"
.format(folder_name, self.account_id))
if saved_states.get(folder_name) != 'finish':
log.info('initializing folder sync')
# STOPSHIP(emfree): replace by appropriate base class.
thread = self.sync_engine_class(self.account_id, folder_name,
folder_id_for[folder_name],
self.email_address,
self.provider_name,
self.poll_frequency,
self.syncmanager_lock,
self.refresh_flags_max,
self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread) or thread.ready():
log.info('folder sync finished/killed',
folder_name=thread.folder_name)
# NOTE: Greenlet is automatically removed from the group.
self.folder_monitors.join()
def test_save_folder_names(db, folder_name_mapping):
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session)
saved_folder_names = {name for name, in
db_session.query(Folder.name).filter(
Folder.account_id == ACCOUNT_ID)}
assert saved_folder_names == {'Inbox', '[Gmail]/Spam',
'[Gmail]/All Mail', '[Gmail]/Sent Mail',
'[Gmail]/Drafts', 'Jobslist', 'Random'}
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with session_scope(ignore_soft_deletes=False) as db_session:
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(self.log, account,
crispin_client.folder_names(), db_session)
Tag.create_canonical_tags(account.namespace, db_session)
folder_id_for = {
name: id_
for id_, name in db_session.query(Folder.id, Folder.name).
filter_by(account_id=self.account_id)
}
saved_states = {
name: state
for name, state in db_session.query(
Folder.name, ImapFolderSyncStatus.state).join(
ImapFolderSyncStatus.folder).filter(
ImapFolderSyncStatus.account_id == self.account_id)
}
for folder_name in sync_folders:
if folder_name not in folder_id_for:
self.log.error("Missing Folder object when starting sync",
folder_name=folder_name,
folder_id_for=folder_id_for)
raise MailsyncError("Missing Folder '{}' on account {}".format(
folder_name, self.account_id))
if saved_states.get(folder_name) != 'finish':
self.log.info('initializing folder sync')
thread = ImapFolderSyncMonitor(
self.account_id, folder_name, folder_id_for[folder_name],
self.email_address, self.provider_name, self.shared_state,
self.folder_state_handlers, self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread):
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread):
self.log.info('folder sync finished')
# NOTE: Greenlet is automatically removed from the group
# after finishing.
self.folder_monitors.join()
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with session_scope(ignore_soft_deletes=False) as db_session:
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(self.log, account,
crispin_client.folder_names(), db_session)
Tag.create_canonical_tags(account.namespace, db_session)
folder_id_for = {name: id_ for id_, name in db_session.query(
Folder.id, Folder.name).filter_by(account_id=self.account_id)}
saved_states = {name: state for name, state in
db_session.query(Folder.name,
ImapFolderSyncStatus.state)
.join(ImapFolderSyncStatus.folder)
.filter(ImapFolderSyncStatus.account_id ==
self.account_id)}
for folder_name in sync_folders:
if folder_name not in folder_id_for:
self.log.error("Missing Folder object when starting sync",
folder_name=folder_name,
folder_id_for=folder_id_for)
raise MailsyncError("Missing Folder '{}' on account {}"
.format(folder_name, self.account_id))
if saved_states.get(folder_name) != 'finish':
self.log.info('initializing folder sync')
thread = ImapFolderSyncMonitor(self.account_id, folder_name,
folder_id_for[folder_name],
self.email_address,
self.provider_name,
self.shared_state,
self.folder_state_handlers,
self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread):
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread):
self.log.info('folder sync finished')
# NOTE: Greenlet is automatically removed from the group
# after finishing.
self.folder_monitors.join()
def poll_impl(self, crispin_client):
log.bind(state='poll')
with mailsync_session_scope() as db_session:
saved_folder_info = common.get_folder_info(
crispin_client.account_id, db_session, self.folder_name)
saved_highestmodseq = saved_folder_info.highestmodseq
# Start a session since we're going to IDLE below anyway...
# This also resets the folder name cache, which we want in order to
# detect folder/label additions and deletions.
status = crispin_client.select_folder(
self.folder_name, uidvalidity_cb(crispin_client.account_id))
log.debug(current_modseq=status['HIGHESTMODSEQ'],
saved_modseq=saved_highestmodseq)
if status['HIGHESTMODSEQ'] > saved_highestmodseq:
with mailsync_session_scope() as db_session:
acc = db_session.query(ImapAccount).get(self.account_id)
save_folder_names(log, acc, crispin_client.folder_names(),
db_session)
self.highestmodseq_update(crispin_client, saved_highestmodseq)
# We really only want to idle on a folder for new messages. Idling on
# `All Mail` won't tell us when messages are archived from the Inbox
if self.folder_name.lower() in IDLE_FOLDERS:
status = crispin_client.select_folder(
self.folder_name, uidvalidity_cb(crispin_client.account_id))
# Idle doesn't pick up flag changes, so we don't want to idle for
# very long, or we won't detect things like messages being marked
# as read.
idle_frequency = 30
log.info('idling', timeout=idle_frequency)
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=idle_frequency)
# If we want to do something with the response, but lousy
# because it uses sequence IDs instead of UIDs
# resp = c.idle_check(timeout=shared_state['poll_frequency'])
# r = dict( EXISTS=[], EXPUNGE=[])
# for msg_uid, cmd in resp:
# r[cmd].append(msg_uid)
# print r
crispin_client.conn.idle_done()
log.info('IDLE triggered poll')
else:
log.info('IDLE sleeping', seconds=self.poll_frequency)
sleep(self.poll_frequency)
def test_save_folder_names(db, folder_name_mapping):
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session)
saved_folder_names = {
name
for name, in db_session.query(Folder.name).filter(
Folder.account_id == ACCOUNT_ID)
}
assert saved_folder_names == {
'Inbox', '[Gmail]/Spam', '[Gmail]/All Mail', '[Gmail]/Sent Mail',
'[Gmail]/Drafts', 'Jobslist', 'Random'
}
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with session_scope() as db_session:
saved_states = dict()
folder_id_for = dict()
for saved_state in db_session.query(ImapFolderSyncStatus)\
.filter_by(account_id=self.account_id):
saved_states[saved_state.folder.name] = saved_state.state
folder_id_for[saved_state.folder.name] = saved_state.folder.id
# it's possible we've never started syncs for these folders before
for folder_id, folder_name, in \
db_session.query(Folder.id, Folder.name).filter_by(
account_id=self.account_id):
folder_id_for[folder_name] = folder_id
with connection_pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(self.log, account,
crispin_client.folder_names(), db_session)
Tag.create_canonical_tags(account.namespace, db_session)
for folder_name in sync_folders:
if saved_states.get(folder_name) != 'finish':
self.log.info("Initializing folder sync for {0}"
.format(folder_name))
thread = ImapFolderSyncMonitor(self.account_id, folder_name,
folder_id_for[folder_name],
self.email_address,
self.provider,
self.shared_state,
self.folder_state_handlers,
self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread):
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread):
self.log.info("Folder sync for {} is done."
.format(folder_name))
# NOTE: Greenlet is automatically removed from the group
# after finishing.
self.folder_monitors.join()
def test_save_folder_names(db, default_account, folder_name_mapping):
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, default_account.id, folder_name_mapping, db_session)
saved_folder_names = {
name for name, in db_session.query(Folder.name).filter(Folder.account_id == default_account.id)
}
assert saved_folder_names == {
"Inbox",
"[Gmail]/Spam",
"[Gmail]/All Mail",
"[Gmail]/Sent Mail",
"[Gmail]/Drafts",
"Jobslist",
"Random",
}
def test_folder_delete_cascades_to_tag(db, folder_name_mapping):
"""Test that when a tag (folder) is deleted, we properly cascade to delete
the Tag object too.
"""
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session)
folders = db_session.query(Folder).filter_by(account_id=ACCOUNT_ID)
assert folders.count() == 7
random_folder = folders.filter_by(name='Random').first()
assert random_folder is not None
random_tag = random_folder.get_associated_tag(db_session)
random_tag_id = random_tag.id
db.session.commit()
folder_name_mapping['extra'] = ['Jobslist']
save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session)
db.session.commit()
random_tag = db_session.query(Tag).get(random_tag_id)
assert random_tag is None
def test_name_collision_folders(db, default_account, folder_name_mapping):
# test that when a user-created folder called 'spam' is created, we don't
# associate it with the canonical spam tag, but instead give it its own
# tag
folder_name_mapping['extra'] = ['spam']
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, default_account.id, folder_name_mapping,
db_session)
spam_tags = db_session.query(Tag).filter_by(
namespace_id=default_account.namespace.id, name='spam')
# There should be one 'Gmail/Spam' canonical tag
assert spam_tags.count() == 1
assert spam_tags.first().public_id == 'spam'
# and one 'imap/spam' non-canonical tag with public_id != 'spam'
spam_tags = db_session.query(Tag).filter_by(
namespace_id=default_account.namespace.id, name='imap/spam')
assert spam_tags.count() == 1
assert spam_tags.first().public_id != 'spam'
# test that when a folder called 'spam' is deleted, we don't delete
# the canonical 'spam' tag
folder_name_mapping['extra'] = []
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, default_account.id, folder_name_mapping,
db_session)
spam_tags = db_session.query(Tag).filter_by(
namespace_id=default_account.namespace.id, name='spam')
# The 'Gmail/Spam' canonical tag should still remain.
assert spam_tags.count() == 1
assert spam_tags.first().public_id == 'spam'
# The 'imap/spam' non-canonical tag shouldn't
spam_tags = db_session.query(Tag).filter_by(
namespace_id=default_account.namespace.id, name='imap/spam')
assert spam_tags.count() == 0
def test_sync_folder_deletes(db, folder_name_mapping):
"""Test that folder deletions properly cascade to deletions of
ImapFoldSyncStatus and ImapFolderInfo."""
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session)
folders = db_session.query(Folder).filter_by(account_id=ACCOUNT_ID)
for folder in folders:
add_imap_status_info_rows(folder.id, ACCOUNT_ID, db_session)
db_session.commit()
assert db_session.query(ImapFolderInfo).count() == 7
assert db_session.query(ImapFolderSyncStatus).count() == 7
folder_name_mapping['extra'] = ['Jobslist']
save_folder_names(log, ACCOUNT_ID, folder_name_mapping, db_session)
saved_folder_names = {name for name, in
db_session.query(Folder.name).filter(
Folder.account_id == ACCOUNT_ID)}
assert saved_folder_names == {'Inbox', '[Gmail]/Spam',
'[Gmail]/All Mail', '[Gmail]/Sent Mail',
'[Gmail]/Drafts', 'Jobslist'}
assert db_session.query(ImapFolderInfo).count() == 6
assert db_session.query(ImapFolderSyncStatus).count() == 6
def test_folder_delete_cascades_to_tag(db, default_account,
folder_name_mapping):
"""Test that when a tag (folder) is deleted, we properly cascade to delete
the Tag object too.
"""
with mailsync_session_scope() as db_session:
log = get_logger()
save_folder_names(log, default_account.id, folder_name_mapping,
db_session)
folders = db_session.query(Folder).filter_by(
account_id=default_account.id)
assert folders.count() == 7
random_folder = folders.filter_by(name='Random').first()
assert random_folder is not None
random_tag = random_folder.get_associated_tag(db_session)
random_tag_id = random_tag.id
db.session.commit()
folder_name_mapping['extra'] = ['Jobslist']
save_folder_names(log, default_account.id, folder_name_mapping,
db_session)
db.session.commit()
random_tag = db_session.query(Tag).get(random_tag_id)
assert random_tag is None
def condstore_base_poll(crispin_client, log, folder_name, shared_state,
highestmodseq_fn):
""" Base polling logic for IMAP servers which support CONDSTORE and IDLE.
The CONDSTORE / HIGHESTMODSEQ mechanism is used to detect new and changed
messages that need syncing.
"""
log.bind(state='poll')
with session_scope(ignore_soft_deletes=False) as db_session:
saved_folder_info = account.get_folder_info(crispin_client.account_id,
db_session, folder_name)
saved_highestmodseq = saved_folder_info.highestmodseq
# Start a session since we're going to IDLE below anyway...
# This also resets the folder name cache, which we want in order to
# detect folder/label additions and deletions.
status = crispin_client.select_folder(
folder_name, uidvalidity_cb(crispin_client.account_id))
log.debug(current_modseq=status['HIGHESTMODSEQ'],
saved_modseq=saved_highestmodseq)
if status['HIGHESTMODSEQ'] > saved_highestmodseq:
with session_scope(ignore_soft_deletes=False) as db_session:
acc = db_session.query(ImapAccount).get(crispin_client.account_id)
save_folder_names(log, acc, crispin_client.folder_names(),
db_session)
highestmodseq_update(crispin_client, log, folder_name,
saved_highestmodseq, highestmodseq_fn,
shared_state['syncmanager_lock'])
# We really only want to idle on a folder for new messages. Idling on
# `All Mail` won't tell us when messages are archived from the Inbox
if folder_name.lower() in IDLE_FOLDERS:
status = crispin_client.select_folder(
folder_name, uidvalidity_cb(crispin_client.account_id))
idle_frequency = 1800 # 30min
log.info('idling', timeout=idle_frequency)
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=idle_frequency)
# If we want to do something with the response, but lousy
# because it uses sequence IDs instead of UIDs
# resp = c.idle_check(timeout=shared_state['poll_frequency'])
# r = dict( EXISTS=[], EXPUNGE=[])
# for msg_uid, cmd in resp:
# r[cmd].append(msg_uid)
# print r
crispin_client.conn.idle_done()
log.info('IDLE triggered poll')
else:
log.info('IDLE sleeping', seconds=shared_state['poll_frequency'])
sleep(shared_state['poll_frequency'])
return 'poll'
