def test_actions_are_claimed(purge_accounts_and_actions, patched_worker):
with session_scope_by_shard_id(0) as db_session:
account = add_generic_imap_account(db_session,
email_address='{}@test.com'.format(0))
schedule_test_action(db_session, account)
with session_scope_by_shard_id(1) as db_session:
account = add_generic_imap_account(db_session,
email_address='{}@test.com'.format(1))
schedule_test_action(db_session, account)
service = SyncbackService(cpu_id=1, total_cpus=2)
service.workers = set()
service._process_log()
gevent.joinall(list(service.workers))
with session_scope_by_shard_id(0) as db_session:
q = db_session.query(ActionLog)
assert q.count() == 1
assert all(a.status == 'pending' for a in q)
with session_scope_by_shard_id(1) as db_session:
q = db_session.query(ActionLog)
assert q.count() == 1
assert all(a.status != 'pending' for a in q)
def test_failed_event_creation(db, patched_syncback_task, default_account,
event):
schedule_action("create_event", event, default_account.namespace.id,
db.session)
schedule_action("update_event", event, default_account.namespace.id,
db.session)
schedule_action("update_event", event, default_account.namespace.id,
db.session)
schedule_action("delete_event", event, default_account.namespace.id,
db.session)
db.session.commit()
NUM_WORKERS = 2
service = SyncbackService(
syncback_id=0,
process_number=0,
total_processes=NUM_WORKERS,
num_workers=NUM_WORKERS,
)
service._restart_workers()
service._process_log()
while not service.task_queue.empty():
gevent.sleep(0.1)
# This has to be a separate while-loop because there's a brief moment where
# the task queue is empty, but num_idle_workers hasn't been updated yet.
# On slower systems, we might need to sleep a bit between the while-loops.
while service.num_idle_workers != NUM_WORKERS:
gevent.sleep(0.1)
q = db.session.query(ActionLog).filter_by(record_id=event.id).all()
assert all(a.status == "failed" for a in q)
def test_actions_are_claimed(purge_accounts_and_actions, patched_task):
with session_scope_by_shard_id(0) as db_session:
account = add_generic_imap_account(
db_session, email_address='{}@test.com'.format(0))
schedule_test_action(db_session, account)
with session_scope_by_shard_id(1) as db_session:
account = add_generic_imap_account(
db_session, email_address='{}@test.com'.format(1))
schedule_test_action(db_session, account)
service = SyncbackService(
syncback_id=0, process_number=1, total_processes=2, num_workers=2)
service._restart_workers()
service._process_log()
while not service.task_queue.empty():
gevent.sleep(0)
with session_scope_by_shard_id(0) as db_session:
q = db_session.query(ActionLog)
assert q.count() == 1
assert all(a.status == 'pending' for a in q)
with session_scope_by_shard_id(1) as db_session:
q = db_session.query(ActionLog)
assert q.count() == 1
assert all(a.status != 'pending' for a in q)
def test_actions_are_claimed(purge_accounts_and_actions, patched_worker):
with session_scope_by_shard_id(0) as db_session:
account = add_generic_imap_account(
db_session, email_address='{}@test.com'.format(0))
schedule_test_action(db_session, account)
with session_scope_by_shard_id(1) as db_session:
account = add_generic_imap_account(
db_session, email_address='{}@test.com'.format(1))
schedule_test_action(db_session, account)
service = SyncbackService(cpu_id=1, total_cpus=2)
service.workers = set()
service._process_log()
gevent.joinall(list(service.workers))
with session_scope_by_shard_id(0) as db_session:
q = db_session.query(ActionLog)
assert q.count() == 1
assert all(a.status == 'pending' for a in q)
with session_scope_by_shard_id(1) as db_session:
q = db_session.query(ActionLog)
assert q.count() == 1
assert all(a.status != 'pending' for a in q)
def test_folder_name_translation(empty_db, generic_account, imap_api_client,
mock_imapclient, monkeypatch):
from inbox.transactions.actions import SyncbackService
syncback = SyncbackService(syncback_id=0, process_number=0,
total_processes=1)
imap_namespaces = (((u'INBOX.', u'.'),),)
mock_imapclient.create_folder = mock.Mock()
mock_imapclient.namespace = mock.Mock(return_value=imap_namespaces)
folder_list = [(('\\HasChildren',), '.', u'INBOX')]
mock_imapclient.list_folders = mock.Mock(return_value=folder_list)
mock_imapclient.has_capability = mock.Mock(return_value=True)
folder_prefix, folder_separator = imap_namespaces[0][0]
generic_account.folder_prefix = folder_prefix
generic_account.folder_separator = folder_separator
empty_db.session.commit()
folder_json = {'display_name': 'Taxes/Accounting'}
imap_api_client.post_data('/folders', folder_json)
syncback._process_log()
gevent.joinall(list(syncback.workers))
mock_imapclient.create_folder.assert_called_with('INBOX.Taxes.Accounting')
def test_folder_name_translation(empty_db, generic_account, imap_api_client,
mock_imapclient, monkeypatch):
from inbox.transactions.actions import SyncbackService
syncback = SyncbackService(syncback_id=0, process_number=0,
total_processes=1, num_workers=2)
imap_namespaces = (((u'INBOX.', u'.'),),)
mock_imapclient.create_folder = mock.Mock()
mock_imapclient.namespace = mock.Mock(return_value=imap_namespaces)
folder_list = [(('\\HasChildren',), '.', u'INBOX')]
mock_imapclient.list_folders = mock.Mock(return_value=folder_list)
mock_imapclient.has_capability = mock.Mock(return_value=True)
folder_prefix, folder_separator = imap_namespaces[0][0]
generic_account.folder_prefix = folder_prefix
generic_account.folder_separator = folder_separator
empty_db.session.commit()
folder_json = {'display_name': 'Taxes/Accounting'}
imap_api_client.post_data('/folders', folder_json)
syncback._process_log()
syncback._restart_workers()
while not syncback.task_queue.empty():
gevent.sleep(0.1)
mock_imapclient.create_folder.assert_called_with('INBOX.Taxes.Accounting')
def test_actions_are_claimed(purge_accounts_and_actions, patched_task):
with session_scope_by_shard_id(0) as db_session:
account = add_generic_imap_account(
db_session, email_address='{}@test.com'.format(0))
schedule_test_action(db_session, account)
with session_scope_by_shard_id(1) as db_session:
account = add_generic_imap_account(
db_session, email_address='{}@test.com'.format(1))
schedule_test_action(db_session, account)
service = SyncbackService(syncback_id=0,
process_number=1,
total_processes=2,
num_workers=2)
service._restart_workers()
service._process_log()
while not service.task_queue.empty():
gevent.sleep(0.1)
with session_scope_by_shard_id(0) as db_session:
q = db_session.query(ActionLog)
assert q.count() == 1
assert all(a.status == 'pending' for a in q)
with session_scope_by_shard_id(1) as db_session:
q = db_session.query(ActionLog)
assert q.count() == 1
assert all(a.status != 'pending' for a in q)
def test_actions_for_invalid_accounts_are_skipped(purge_accounts_and_actions,
patched_task):
with session_scope_by_shard_id(0) as db_session:
account = add_generic_imap_account(db_session,
email_address='*****@*****.**')
schedule_test_action(db_session, account)
namespace_id = account.namespace.id
count = db_session.query(ActionLog).filter(
ActionLog.namespace_id == namespace_id).count()
assert account.sync_state != 'invalid'
another_account = add_generic_imap_account(
db_session, email_address='*****@*****.**')
schedule_test_action(db_session, another_account)
another_namespace_id = another_account.namespace.id
another_count = db_session.query(ActionLog).filter(
ActionLog.namespace_id == another_namespace_id).count()
assert another_account.sync_state != 'invalid'
account.mark_invalid()
db_session.commit()
service = SyncbackService(syncback_id=0,
process_number=0,
total_processes=2,
num_workers=2)
service._process_log()
while not service.task_queue.empty():
gevent.sleep(0)
with session_scope_by_shard_id(0) as db_session:
q = db_session.query(ActionLog).filter(
ActionLog.namespace_id == namespace_id,
ActionLog.status == 'pending')
assert q.count() == count
q = db_session.query(ActionLog).filter(
ActionLog.namespace_id == another_namespace_id)
assert q.filter(ActionLog.status == 'pending').count() == 0
assert q.filter(
ActionLog.status == 'successful').count() == another_count
def test_actions_claimed_by_a_single_service(purge_accounts_and_actions,
patched_worker):
actionlogs = []
for key in (0, 1):
with session_scope_by_shard_id(key) as db_session:
account = make_imap_account(db_session, '{}@test.com'.format(key))
schedule_test_action(db_session, account)
actionlogs += [db_session.query(ActionLog).one().id]
services = []
for cpu_id in (0, 1):
service = SyncbackService(cpu_id=cpu_id, total_cpus=2)
service.workers = set()
service._process_log()
services.append(service)
for i, service in enumerate(services):
assert len(service.workers) == 1
assert list(service.workers)[0].action_log_id == actionlogs[i]
gevent.joinall(list(service.workers))
def test_actions_for_invalid_accounts_are_skipped(purge_accounts_and_actions,
patched_worker):
with session_scope_by_shard_id(0) as db_session:
account = add_generic_imap_account(
db_session, email_address='*****@*****.**')
schedule_test_action(db_session, account)
namespace_id = account.namespace.id
count = db_session.query(ActionLog).filter(
ActionLog.namespace_id == namespace_id).count()
assert account.sync_state != 'invalid'
another_account = add_generic_imap_account(
db_session, email_address='*****@*****.**')
schedule_test_action(db_session, another_account)
another_namespace_id = another_account.namespace.id
another_count = db_session.query(ActionLog).filter(
ActionLog.namespace_id == another_namespace_id).count()
assert another_account.sync_state != 'invalid'
account.mark_invalid()
db_session.commit()
service = SyncbackService(
syncback_id=0, process_number=0, total_processes=2)
service._process_log()
while len(service.workers) >= 1:
gevent.sleep(0.1)
gevent.killall(service.workers)
with session_scope_by_shard_id(0) as db_session:
q = db_session.query(ActionLog).filter(
ActionLog.namespace_id == namespace_id,
ActionLog.status == 'pending')
assert q.count() == count
q = db_session.query(ActionLog).filter(
ActionLog.namespace_id == another_namespace_id)
assert q.filter(ActionLog.status == 'pending').count() == 0
assert q.filter(ActionLog.status == 'successful').count() == another_count
def test_actions_claimed_by_a_single_service(purge_accounts_and_actions,
patched_task):
actionlogs = []
for key in (0, 1):
with session_scope_by_shard_id(key) as db_session:
account = add_generic_imap_account(
db_session,
email_address='{}@test.com'.format(key))
schedule_test_action(db_session, account)
actionlogs += [db_session.query(ActionLog).one().id]
services = []
for process_number in (0, 1):
service = SyncbackService(
syncback_id=0, process_number=process_number, total_processes=2,
num_workers=2)
service._process_log()
services.append(service)
for i, service in enumerate(services):
assert service.task_queue.qsize() == 1
assert service.task_queue.peek().action_log_ids() == [actionlogs[i]]
def test_actions_claimed_by_a_single_service(purge_accounts_and_actions,
patched_task):
actionlogs = []
for key in (0, 1):
with session_scope_by_shard_id(key) as db_session:
account = add_generic_imap_account(
db_session, email_address='{}@test.com'.format(key))
schedule_test_action(db_session, account)
actionlogs += [db_session.query(ActionLog).one().id]
services = []
for process_number in (0, 1):
service = SyncbackService(syncback_id=0,
process_number=process_number,
total_processes=2,
num_workers=2)
service._process_log()
services.append(service)
for i, service in enumerate(services):
assert service.task_queue.qsize() == 1
assert service.task_queue.peek().action_log_ids() == [actionlogs[i]]
def test_failed_event_creation(db, patched_syncback_task, default_account, event):
schedule_action('create_event', event, default_account.namespace.id, db.session)
schedule_action('update_event', event, default_account.namespace.id, db.session)
schedule_action('update_event', event, default_account.namespace.id, db.session)
schedule_action('delete_event', event, default_account.namespace.id, db.session)
db.session.commit()
NUM_WORKERS = 2
service = SyncbackService(syncback_id=0, process_number=0,
total_processes=NUM_WORKERS, num_workers=NUM_WORKERS)
service._restart_workers()
service._process_log()
while not service.task_queue.empty():
gevent.sleep(0.1)
# This has to be a separate while-loop because there's a brief moment where
# the task queue is empty, but num_idle_workers hasn't been updated yet.
# On slower systems, we might need to sleep a bit between the while-loops.
while service.num_idle_workers != NUM_WORKERS:
gevent.sleep(0.1)
q = db.session.query(ActionLog).filter_by(record_id=event.id).all()
assert all(a.status == 'failed' for a in q)
