def test_start_new_accounts_when_stealing_enabled(db, default_account):
default_account.sync_host = None
db.session.commit()
ss = SyncService(cpu_id=1, total_cpus=2)
assert ss.accounts_to_start() == [1]
db.session.expire_all()
assert default_account.sync_host == platform.node()
def test_concurrent_syncs(db, default_account, config):
ss1 = SyncService(cpu_id=1, total_cpus=2)
ss2 = SyncService(cpu_id=1, total_cpus=2)
ss2.host = 'other-host'
# Check that only one SyncService instance claims the account.
assert ss1.accounts_to_start() == [1]
assert ss2.accounts_to_start() == []
def patched_sync_service(mock_queue_client, host=host, cpu_id=0):
s = SyncService(process_identifier='{}:{}'.format(host, cpu_id),
cpu_id=cpu_id)
s.queue_client = mock_queue_client
s.start_sync = mock.Mock(
side_effect=lambda aid: s.syncing_accounts.add(aid))
return s
def patched_sync_service(mock_queue_client, host=host, cpu_id=0):
s = SyncService(process_identifier='{}:{}'.format(host, cpu_id),
cpu_id=cpu_id)
s.queue_client = mock_queue_client
s.start_sync = mock.Mock(
side_effect=lambda aid: s.syncing_accounts.add(aid))
return s
def test_start_new_accounts_when_stealing_enabled(db, default_account):
default_account.sync_host = None
db.session.commit()
ss = SyncService(cpu_id=1, total_cpus=2)
assert ss.accounts_to_start() == [1]
db.session.expire_all()
assert default_account.sync_host == platform.node()
def test_concurrent_syncs(db, default_account, config):
purge_other_accounts(default_account)
ss1 = SyncService(cpu_id=default_account.id % 2, total_cpus=2)
ss2 = SyncService(cpu_id=default_account.id % 2, total_cpus=2)
ss2.host = "other-host"
# Check that only one SyncService instance claims the account.
assert ss1.accounts_to_start() == [default_account.id]
assert ss2.accounts_to_start() == []
def test_dont_start_new_accounts_when_stealing_disabled(
db, config, default_account):
default_account.sync_host = None
db.session.commit()
config['SYNC_STEAL_ACCOUNTS'] = False
ss = SyncService(cpu_id=1, total_cpus=2)
assert ss.accounts_to_start() == []
assert default_account.sync_host is None
def test_dont_start_new_accounts_when_stealing_disabled(db, config, default_account):
purge_other_accounts(default_account)
default_account.sync_host = None
db.session.commit()
config["SYNC_STEAL_ACCOUNTS"] = False
ss = SyncService(cpu_id=default_account.id % 2, total_cpus=2)
assert ss.accounts_to_start() == []
assert default_account.sync_host is None
def patched_sync_service(db, host=host, process_number=0):
s = SyncService(process_identifier="{}:{}".format(host, process_number), process_number=process_number)
def start_sync(aid):
acc = db.session.query(Account).get(aid)
acc.sync_host = s.process_identifier
acc.sync_started()
s.syncing_accounts.add(aid)
db.session.commit()
return True
s.start_sync = mock.Mock(side_effect=start_sync)
return s
def test_dont_start_disabled_accounts(db, config, default_account):
purge_other_accounts(default_account)
config['SYNC_STEAL_ACCOUNTS'] = True
default_account.sync_host = None
default_account.disable_sync(reason='testing')
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss.accounts_to_start() == []
assert default_account.sync_host is None
assert default_account.sync_should_run is False
default_account.sync_host = platform.node()
default_account.disable_sync('testing')
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss.accounts_to_start() == []
assert default_account.sync_should_run is False
# Invalid Credentials
default_account.mark_invalid()
default_account.sync_host = None
db.session.commit()
# Don't steal invalid accounts
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss.accounts_to_start() == []
# Don't explicitly start invalid accounts
default_account.sync_host = platform.node()
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss.accounts_to_start() == []
def patched_sync_service(db, mock_queue_client, host=host, cpu_id=0):
s = SyncService(process_identifier='{}:{}'.format(host, cpu_id),
cpu_id=cpu_id)
s.queue_client = mock_queue_client
def start_sync(aid):
acc = db.session.query(Account).get(aid)
acc.sync_host = s.process_identifier
acc.sync_started()
s.syncing_accounts.add(aid)
db.session.commit()
s.start_sync = mock.Mock(side_effect=start_sync)
return s
def patched_sync_service(db, mock_queue_client, host=host, process_number=0):
s = SyncService(process_identifier='{}:{}'.format(host, process_number),
process_number=process_number)
s.queue_client = mock_queue_client
def start_sync(aid):
acc = db.session.query(Account).get(aid)
acc.sync_host = s.process_identifier
acc.sync_started()
s.syncing_accounts.add(aid)
db.session.commit()
s.start_sync = mock.Mock(side_effect=start_sync)
return s
def test_concurrent_syncs(db, default_account, config):
ss1 = SyncService(cpu_id=1, total_cpus=2)
ss2 = SyncService(cpu_id=1, total_cpus=2)
ss2.host = 'other-host'
# Check that only one SyncService instance claims the account.
assert ss1.accounts_to_start() == [1]
assert ss2.accounts_to_start() == []
def test_sync_transitions(db, default_account, config):
purge_other_accounts(default_account)
ss = SyncService(cpu_id=default_account.id % 2, total_cpus=2)
default_account.enable_sync()
db.session.commit()
assert ss.accounts_to_start() == [default_account.id]
default_account.disable_sync("manual")
db.session.commit()
assert ss.accounts_to_start() == []
assert default_account.sync_should_run is False
assert default_account._sync_status["sync_disabled_reason"] == "manual"
default_account.mark_invalid()
db.session.commit()
assert ss.accounts_to_start() == []
assert default_account.sync_state == "invalid"
assert default_account._sync_status["sync_disabled_reason"] == "invalid credentials"
assert default_account.sync_should_run is False
def test_sync_transitions(db, default_account, config):
ss = SyncService(cpu_id=1, total_cpus=2)
default_account.enable_sync()
db.session.commit()
assert ss.accounts_to_start() == [1]
default_account.disable_sync('manual')
db.session.commit()
assert ss.accounts_to_start() == []
assert default_account.sync_should_run is False
assert default_account._sync_status['sync_disabled_reason'] == 'manual'
default_account.mark_invalid()
db.session.commit()
assert ss.accounts_to_start() == []
assert default_account.sync_state == 'invalid'
assert default_account._sync_status['sync_disabled_reason'] == \
'invalid credentials'
assert default_account.sync_should_run is False
def test_sync_transitions(db, default_account, config):
ss = SyncService(cpu_id=1, total_cpus=2)
default_account.enable_sync()
db.session.commit()
assert ss.accounts_to_start() == [1]
default_account.disable_sync('manual')
db.session.commit()
assert ss.accounts_to_start() == []
assert default_account.sync_should_run is False
assert default_account._sync_status['sync_disabled_reason'] == 'manual'
default_account.mark_invalid()
db.session.commit()
assert ss.accounts_to_start() == []
assert default_account.sync_state == 'invalid'
assert default_account._sync_status['sync_disabled_reason'] == \
'invalid credentials'
assert default_account.sync_should_run is False
def test_stealing_limited_by_host(db, config):
host = platform.node()
config['DATABASE_HOSTS'][0]['SHARDS'][0]['SYNC_HOSTS'] = [host]
config['DATABASE_HOSTS'][0]['SHARDS'][1]['SYNC_HOSTS'] = ['otherhost']
purge_other_accounts()
ss = SyncService(cpu_id=0, total_cpus=1)
for key in (0, 1):
with session_scope_by_shard_id(key) as db_session:
acc = Account()
acc.namespace = Namespace()
db_session.add(acc)
db_session.commit()
ss.accounts_to_start()
with session_scope_by_shard_id(0) as db_session:
acc = db_session.query(Account).first()
assert acc.sync_host == host
with session_scope_by_shard_id(1) as db_session:
acc = db_session.query(Account).first()
assert acc.sync_host is None
def test_stealing_limited_by_host(db, config):
host = platform.node()
config['DATABASE_HOSTS'][0]['SHARDS'][0]['SYNC_HOSTS'] = [host]
config['DATABASE_HOSTS'][0]['SHARDS'][1]['SYNC_HOSTS'] = ['otherhost']
purge_other_accounts()
ss = SyncService(cpu_id=0, total_cpus=1)
for key in (0, 1):
with session_scope_by_shard_id(key) as db_session:
acc = Account()
acc.namespace = Namespace()
db_session.add(acc)
db_session.commit()
ss.accounts_to_start()
with session_scope_by_shard_id(0) as db_session:
acc = db_session.query(Account).first()
assert acc.sync_host == host
with session_scope_by_shard_id(1) as db_session:
acc = db_session.query(Account).first()
assert acc.sync_host is None
def test_accounts_started_on_all_shards(db, default_account, config):
config['SYNC_STEAL_ACCOUNTS'] = True
purge_other_accounts(default_account)
default_account.sync_host = None
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
ss.host = 'localhost'
account_ids = {default_account.id}
for key in (0, 1):
with session_scope_by_shard_id(key) as db_session:
acc = Account()
acc.namespace = Namespace()
db_session.add(acc)
db_session.commit()
account_ids.add(acc.id)
assert len(account_ids) == 3
assert set(ss.accounts_to_start()) == account_ids
for id_ in account_ids:
with session_scope(id_) as db_session:
acc = db_session.query(Account).get(id_)
assert acc.sync_host == 'localhost'
def test_accounts_started_on_all_shards(db, default_account, config):
config['SYNC_STEAL_ACCOUNTS'] = True
purge_other_accounts(default_account)
default_account.sync_host = None
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
ss.host = 'localhost'
account_ids = {default_account.id}
for key in (0, 1):
with session_scope_by_shard_id(key) as db_session:
acc = Account()
acc.namespace = Namespace()
db_session.add(acc)
db_session.commit()
account_ids.add(acc.id)
assert len(account_ids) == 3
assert set(ss.accounts_to_start()) == account_ids
for id_ in account_ids:
with session_scope(id_) as db_session:
acc = db_session.query(Account).get(id_)
assert acc.sync_host == 'localhost'
def test_start_already_assigned_accounts(db, default_account):
purge_other_accounts(default_account)
default_account.sync_host = platform.node()
ss = SyncService(cpu_id=default_account.id % 2, total_cpus=2)
assert ss.accounts_to_start() == [default_account.id]
def test_start_already_assigned_accounts(db, default_account):
default_account.sync_host = platform.node()
ss = SyncService(cpu_id=1, total_cpus=2)
assert ss.accounts_to_start() == [1]
def test_sync_start(db, default_account, config):
# Make sure having fqdn set locally gets assigned to us
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss._get_local_accounts() == [1]
# Not from other cpus
ss = SyncService(cpu_id=1, total_cpus=1)
assert ss._get_local_accounts() == []
# Different host
default_account.sync_host = "some-random-host"
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss._get_local_accounts() == []
# Explicit
default_account.sync_host = platform.node()
db.session.commit()
assert ss._get_local_accounts() == [1]
default_account.sync_host = None
db.session.commit()
# No host, work stealing enabled
config['SYNC_STEAL_ACCOUNTS'] = True
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss._get_local_accounts() == [1]
# No host, no work stealing disabled
config['SYNC_STEAL_ACCOUNTS'] = False
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss._get_local_accounts() == []
default_account.sync_state = 'stopped'
# Stopped
default_account.sync_host = None
db.session.commit()
# Don't steal stopped accounts
config['SYNC_STEAL_ACCOUNTS'] = True
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss._get_local_accounts() == []
# Don't explicitly start stopped accounts
default_account.sync_host = platform.node()
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss._get_local_accounts() == []
# Invalid Credentials
default_account.sync_state = 'invalid'
db.session.commit()
# Don't steal invalid accounts
config['SYNC_STEAL_ACCOUNTS'] = True
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss._get_local_accounts() == []
# Don't explicitly start invalid accounts
default_account.sync_host = platform.node()
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss._get_local_accounts() == []
def test_dont_start_disabled_accounts(db, config, default_account):
purge_other_accounts(default_account)
config['SYNC_STEAL_ACCOUNTS'] = True
default_account.sync_host = None
default_account.disable_sync(reason='testing')
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss.accounts_to_start() == []
assert default_account.sync_host is None
assert default_account.sync_should_run is False
default_account.sync_host = platform.node()
default_account.disable_sync('testing')
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss.accounts_to_start() == []
assert default_account.sync_should_run is False
# Invalid Credentials
default_account.mark_invalid()
default_account.sync_host = None
db.session.commit()
# Don't steal invalid accounts
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss.accounts_to_start() == []
# Don't explicitly start invalid accounts
default_account.sync_host = platform.node()
db.session.commit()
ss = SyncService(cpu_id=0, total_cpus=1)
assert ss.accounts_to_start() == []
def test_dont_start_accounts_on_other_host(db, default_account):
purge_other_accounts(default_account)
default_account.sync_host = 'other-host'
db.session.commit()
ss = SyncService(cpu_id=1, total_cpus=2)
assert ss.accounts_to_start() == []
def test_dont_start_accounts_for_other_cpus(db, default_account):
purge_other_accounts(default_account)
default_account.sync_host = platform.node()
ss = SyncService(cpu_id=default_account.id + 1, total_cpus=2**22)
assert ss.accounts_to_start() == []
def test_start_already_assigned_accounts(db, default_account):
purge_other_accounts(default_account)
default_account.sync_host = platform.node()
ss = SyncService(cpu_id=default_account.id % 2, total_cpus=2)
assert ss.accounts_to_start() == [default_account.id]
def test_start_already_assigned_accounts(db, default_account):
default_account.sync_host = platform.node()
ss = SyncService(cpu_id=1, total_cpus=2)
assert ss.accounts_to_start() == [1]
def main(prod, enable_tracer, enable_profiler, config, process_num,
exit_after):
""" Launch the Nylas sync service. """
level = os.environ.get("LOGLEVEL", inbox_config.get("LOGLEVEL"))
configure_logging(log_level=level)
reconfigure_logging()
maybe_enable_rollbar()
if config is not None:
from inbox.util.startup import load_overrides
config_path = os.path.abspath(config)
load_overrides(config_path)
if not prod:
preflight()
total_processes = int(os.environ.get("MAILSYNC_PROCESSES", 1))
setproctitle.setproctitle("sync-engine-{}".format(process_num))
log = get_logger()
log.info(
"start",
components=["mail sync", "contact sync", "calendar sync"],
host=platform.node(),
process_num=process_num,
total_processes=total_processes,
recursion_limit=sys.getrecursionlimit(),
)
print(banner, file=sys.stderr)
print(file=sys.stderr)
print("Python", sys.version, file=sys.stderr)
if enable_profiler:
inbox_config["DEBUG_PROFILING_ON"] = True
port = 16384 + process_num
enable_profiler_api = inbox_config.get("DEBUG_PROFILING_ON")
process_identifier = "{}:{}".format(platform.node(), process_num)
if exit_after:
exit_after = exit_after.split(":")
exit_after_min, exit_after_max = int(exit_after[0]), int(exit_after[1])
else:
exit_after_min, exit_after_max = None, None
sync_service = SyncService(
process_identifier,
process_num,
exit_after_min=exit_after_min,
exit_after_max=exit_after_max,
)
signal.signal(signal.SIGTERM, sync_service.stop)
signal.signal(signal.SIGINT, sync_service.stop)
http_frontend = SyncHTTPFrontend(sync_service, port, enable_tracer,
enable_profiler_api)
sync_service.register_pending_avgs_provider(http_frontend)
http_frontend.start()
sync_service.run()
print("\033[94mNylas Sync Engine exiting...\033[0m", file=sys.stderr)
def test_dont_start_accounts_on_other_host(db, default_account):
purge_other_accounts(default_account)
default_account.sync_host = 'other-host'
db.session.commit()
ss = SyncService(cpu_id=1, total_cpus=2)
assert ss.accounts_to_start() == []
def test_dont_start_accounts_for_other_cpus(db, default_account):
purge_other_accounts(default_account)
default_account.sync_host = platform.node()
ss = SyncService(cpu_id=default_account.id + 1, total_cpus=2**22)
assert ss.accounts_to_start() == []