def test_mutable_json_type(db, config, default_account, folder):
"""
Test that FolderSync._sync_status which is a mutable JSON column is
updated as expected.
"""
from inbox.models.backends.imap import ImapFolderSyncStatus
sync_status = ImapFolderSyncStatus(account_id=default_account.id, folder=folder)
db.session.add(sync_status)
db.session.commit()
original_metrics = sync_status.metrics
metrics = dict(download_uid_count=10, queue_checked_at=datetime.utcnow())
sync_status.update_metrics(metrics)
updated_metrics = sync_status.metrics
metrics.update(original_metrics)
assert updated_metrics != original_metrics and updated_metrics == metrics, "metrics not updated correctly"
# Reupdate status
new_metrics = dict(delete_uid_count=50, download_uid_count=100, queue_checked_at=datetime.utcnow())
sync_status.update_metrics(new_metrics)
latest_metrics = sync_status.metrics
metrics.update(new_metrics)
assert latest_metrics == metrics, "metrics not re-updated correctly"
def _load_state(self):
with mailsync_session_scope() as db_session:
try:
state = ImapFolderSyncStatus.state
saved_folder_status = db_session.query(ImapFolderSyncStatus)\
.filter_by(account_id=self.account_id,
folder_id=self.folder_id).options(
load_only(state)).one()
except NoResultFound:
saved_folder_status = ImapFolderSyncStatus(
account_id=self.account_id, folder_id=self.folder_id)
db_session.add(saved_folder_status)
saved_folder_status.start_sync()
db_session.commit()
self.state = saved_folder_status.state
return saved_folder_status
def _load_state(self):
with session_scope(self.namespace_id) as db_session:
try:
state = ImapFolderSyncStatus.state
saved_folder_status = db_session.query(ImapFolderSyncStatus)\
.filter_by(account_id=self.account_id,
folder_id=self.folder_id).options(
load_only(state)).one()
except NoResultFound:
saved_folder_status = ImapFolderSyncStatus(
account_id=self.account_id, folder_id=self.folder_id)
db_session.add(saved_folder_status)
saved_folder_status.start_sync()
db_session.commit()
self.state = saved_folder_status.state
return saved_folder_status
def _run_impl(self):
# We do NOT ignore soft deletes in the mail sync because it gets real
# complicated handling e.g. when backends reuse imapids. ImapUid
# objects are the only objects deleted by the mail sync backends
# anyway.
with session_scope(ignore_soft_deletes=False) as db_session:
try:
state = ImapFolderSyncStatus.state
saved_folder_status = db_session.query(ImapFolderSyncStatus)\
.filter_by(account_id=self.account_id,
folder_id=self.folder_id).options(
load_only(state)).one()
except NoResultFound:
saved_folder_status = ImapFolderSyncStatus(
account_id=self.account_id, folder_id=self.folder_id)
db_session.add(saved_folder_status)
saved_folder_status.start_sync()
db_session.commit()
self.state = saved_folder_status.state
# NOTE: The parent ImapSyncMonitor handler could kill us at any
# time if it receives a shutdown command. The shutdown command is
# equivalent to ctrl-c.
while True:
old_state = self.state
try:
self.state = self.state_handlers[old_state](self.conn_pool,
self.log,
self.folder_name,
self.shared_state)
except UidInvalid:
self.state = self.state + ' uidinvalid'
# State handlers are idempotent, so it's okay if we're
# killed between the end of the handler and the commit.
if self.state != old_state:
# Don't need to re-query, will auto refresh on re-associate.
with session_scope(ignore_soft_deletes=False) as db_session:
db_session.add(saved_folder_status)
saved_folder_status.state = self.state
db_session.commit()
if self.state == 'finish':
return
def _run_impl(self):
# We do NOT ignore soft deletes in the mail sync because it gets real
# complicated handling e.g. when backends reuse imapids. ImapUid
# objects are the only objects deleted by the mail sync backends
# anyway.
with session_scope(ignore_soft_deletes=False) as db_session:
try:
state = ImapFolderSyncStatus.state
saved_folder_status = db_session.query(ImapFolderSyncStatus)\
.filter_by(account_id=self.account_id,
folder_id=self.folder_id).options(
load_only(state)).one()
except NoResultFound:
saved_folder_status = ImapFolderSyncStatus(
account_id=self.account_id, folder_id=self.folder_id)
db_session.add(saved_folder_status)
saved_folder_status.start_sync()
db_session.commit()
self.state = saved_folder_status.state
# NOTE: The parent ImapSyncMonitor handler could kill us at any
# time if it receives a shutdown command. The shutdown command is
# equivalent to ctrl-c.
while True:
old_state = self.state
try:
self.state = self.state_handlers[old_state](
self.conn_pool, self.log, self.folder_name,
self.shared_state)
except UidInvalid:
self.state = self.state + ' uidinvalid'
# State handlers are idempotent, so it's okay if we're
# killed between the end of the handler and the commit.
if self.state != old_state:
# Don't need to re-query, will auto refresh on re-associate.
with session_scope(ignore_soft_deletes=False) as db_session:
db_session.add(saved_folder_status)
saved_folder_status.state = self.state
db_session.commit()
if self.state == 'finish':
return
def _run_impl(self):
# We do NOT ignore soft deletes in the mail sync because it gets real
# complicated handling e.g. when backends reuse imapids. ImapUid
# objects are the only objects deleted by the mail sync backends
# anyway.
with session_scope(ignore_soft_deletes=False) as db_session:
try:
saved_folder_status = db_session.query(ImapFolderSyncStatus)\
.filter_by(account_id=self.account_id,
folder_id=self.folder_id).one()
except NoResultFound:
saved_folder_status = ImapFolderSyncStatus(
account_id=self.account_id, folder_id=self.folder_id)
db_session.add(saved_folder_status)
saved_folder_status.update_metrics(
dict(run_state='running',
sync_start_time=datetime.utcnow(),
sync_end_time=None,
sync_error=None))
db_session.commit()
self.state = saved_folder_status.state
# NOTE: The parent ImapSyncMonitor handler could kill us at any
# time if it receives a shutdown command. The shutdown command is
# equivalent to ctrl-c.
while True:
try:
self.state = saved_folder_status.state = \
self.state_handlers[saved_folder_status.state](
self.conn_pool, db_session, self.log,
self.folder_name, self.shared_state)
except UidInvalid:
self.state = saved_folder_status.state = \
self.state + ' uidinvalid'
# State handlers are idempotent, so it's okay if we're
# killed between the end of the handler and the commit.
db_session.commit()
if self.state == 'finish':
return
def test_mutable_json_type(db, config, default_account, folder):
"""
Test that FolderSync._sync_status which is a mutable JSON column is
updated as expected.
"""
from inbox.models.backends.imap import ImapFolderSyncStatus
sync_status = ImapFolderSyncStatus(account_id=default_account.id,
folder=folder)
db.session.add(sync_status)
db.session.commit()
original_metrics = sync_status.metrics
metrics = dict(download_uid_count=10, queue_checked_at=datetime.utcnow())
sync_status.update_metrics(metrics)
updated_metrics = sync_status.metrics
metrics.update(original_metrics)
assert updated_metrics != original_metrics and updated_metrics == metrics,\
'metrics not updated correctly'
# Reupdate status
new_metrics = dict(delete_uid_count=50,
download_uid_count=100,
queue_checked_at=datetime.utcnow())
sync_status.update_metrics(new_metrics)
latest_metrics = sync_status.metrics
metrics.update(new_metrics)
assert latest_metrics == metrics, 'metrics not re-updated correctly'
def add_imap_status_info_rows(folder_id, account_id, db_session):
"""Add placeholder ImapFolderSyncStatus and ImapFolderInfo rows for this
folder_id if none exist."""
if not db_session.query(ImapFolderSyncStatus).filter_by(
account_id=account_id, folder_id=folder_id).all():
db_session.add(
ImapFolderSyncStatus(account_id=ACCOUNT_ID,
folder_id=folder_id,
state='initial'))
if not db_session.query(ImapFolderInfo).filter_by(
account_id=account_id, folder_id=folder_id).all():
db_session.add(
ImapFolderInfo(account_id=account_id,
folder_id=folder_id,
uidvalidity=1,
highestmodseq=22))
def update_folder_sync_status(self, cb):
# Loads the folder sync status and invokes the provided callback to
# modify it. Commits any changes and updates `self.state` to ensure
# they are never out of sync.
with session_scope(self.namespace_id) as db_session:
try:
state = ImapFolderSyncStatus.state
saved_folder_status = db_session.query(ImapFolderSyncStatus)\
.filter_by(account_id=self.account_id, folder_id=self.folder_id)\
.options(load_only(state)).one()
except NoResultFound:
saved_folder_status = ImapFolderSyncStatus(
account_id=self.account_id, folder_id=self.folder_id)
db_session.add(saved_folder_status)
cb(saved_folder_status)
db_session.commit()
self.state = saved_folder_status.state
def create_folder_with_syncstatus(account, name, canonical_name, db_session):
folder = Folder.find_or_create(db_session, account, name, canonical_name)
folder.imapsyncstatus = ImapFolderSyncStatus(account=account)
db_session.commit()
return folder