def check_delayed_and_suppressed(plan_name: str):
sdk_plan.wait_for_plan_status(foldered_name, plan_name, "DELAYED")
sdk_metrics.wait_for_scheduler_gauge_value(
foldered_name,
"is_suppressed",
lambda result: isinstance(result, bool) and bool(
result), # Should be set to true.
)
def test_add_deploy_restart_remove():
svc1 = "test1"
# add svc as test1:
sdk_cmd.service_request("POST",
config.SERVICE_NAME,
"/v1/multi/{}?yaml=svc".format(svc1),
json=service_params(svc1))
# get list, should immediately have new entry:
service = get_service_list()[0]
assert service["service"] == svc1
assert service["yaml"] == "svc"
assert not service["uninstall"]
sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc1)
task_ids = sdk_tasks.get_task_ids("marathon", config.SERVICE_NAME)
log.info("list of task ids {}".format(task_ids))
old_task_id = task_ids[0]
# restart and check that service is recovered:
sdk_marathon.restart_app(config.SERVICE_NAME)
# check that scheduler task was relaunched
sdk_tasks.check_scheduler_relaunched(config.SERVICE_NAME, old_task_id)
service = wait_for_service_count(1)[0]
assert service["service"] == svc1
assert service["yaml"] == "svc"
assert not service["uninstall"]
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc1)
# verify that svc.yml was deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == [
"hello-0:[server]",
"world-0:[server]",
"world-1:[server]",
]
# trigger service removal, wait for removal:
sdk_cmd.service_request("DELETE", config.SERVICE_NAME,
"/v1/multi/{}".format(svc1))
# check delete bit is set. however, be permissive of service being removed VERY quickly:
services = get_service_list()
assert len(services) <= 1
for service in services:
assert service["service"] == svc1
assert service["yaml"] == "svc"
assert service["uninstall"]
wait_for_service_count(0)
def test_add_deploy_restart_remove():
svc1 = 'test1'
# add svc as test1:
sdk_cmd.service_request('POST',
config.SERVICE_NAME,
'/v1/multi/{}?yaml=svc'.format(svc1),
json=service_params(svc1))
# get list, should immediately have new entry:
service = get_service_list()[0]
assert service['service'] == svc1
assert service['yaml'] == 'svc'
assert not service['uninstall']
sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
'deploy',
'COMPLETE',
multiservice_name=svc1)
task_ids = sdk_tasks.get_task_ids('marathon', config.SERVICE_NAME)
log.info('list of task ids {}'.format(task_ids))
old_task_id = task_ids[0]
# restart and check that service is recovered:
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_marathon.wait_for_app_running(config.SERVICE_NAME,
sdk_marathon.TIMEOUT_SECONDS)
# check that scheduler task was relaunched
check_scheduler_relaunched(config.SERVICE_NAME, old_task_id)
service = wait_for_service_count(1)[0]
assert service['service'] == svc1
assert service['yaml'] == 'svc'
assert not service['uninstall']
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
'deploy',
'COMPLETE',
multiservice_name=svc1)
# verify that svc.yml was deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == [
'hello-0:[server]', 'world-0:[server]', 'world-1:[server]'
]
# trigger service removal, wait for removal:
sdk_cmd.service_request('DELETE', config.SERVICE_NAME,
'/v1/multi/{}'.format(svc1))
# check delete bit is set. however, be permissive of service being removed VERY quickly:
services = get_service_list()
assert len(services) <= 1
for service in services:
assert service['service'] == svc1
assert service['yaml'] == 'svc'
assert service['uninstall']
wait_for_service_count(0)
def test_region_config_update_does_not_succeed(configure_universe,
local_service):
change_region_config(REMOTE_REGION)
sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
'deploy',
'ERROR',
timeout_seconds=180)
change_region_config(None)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME,
timeout_seconds=180)
def test_default_plan_backoff():
sdk_install.install(
config.PACKAGE_NAME,
foldered_name,
expected_running_tasks=0,
additional_options=back_off_crash_loop_options,
wait_for_deployment=False,
wait_for_all_conditions=False,
)
# State transition should be STARTING -> STARTED -> DELAYED in a loop.
# As STARTING lasts for a very short duration, we test the switch between other two states.
check_delayed_and_suppressed("deploy")
sdk_plan.wait_for_plan_status(foldered_name, "deploy", "STARTED")
check_delayed_and_suppressed("deploy")
def test_add_multiple_uninstall():
# add two services:
svc1 = 'test1'
sdk_cmd.service_request('POST', config.SERVICE_NAME, '/v1/multi/{}?yaml=svc'.format(svc1), json=service_params(svc1))
svc2 = 'test2'
sdk_cmd.service_request('POST', config.SERVICE_NAME, '/v1/multi/{}?yaml=simple'.format(svc2), json=service_params(svc2))
# get list, should immediately have new entries:
services = get_service_list()
assert len(services) == 2
for service in services:
name = service['service']
assert name in (svc1, svc2)
if name == svc1:
assert service['yaml'] == 'svc'
else:
assert service['yaml'] == 'simple'
assert not service['uninstall']
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'COMPLETE', multiservice_name=svc1)
# verify that svc.yml was deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == ['hello-0:[server]', 'world-0:[server]', 'world-1:[server]']
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'COMPLETE', multiservice_name=svc2)
# verify that simple.yml was deployed as svc2:
assert sdk_plan.get_all_step_names(plan) == ['hello-0:[server]']
# remove one service, then immediately restart app to verify recovery during service removal:
sdk_cmd.service_request('DELETE', config.SERVICE_NAME, '/v1/multi/{}'.format(svc2))
# check delete bits is set. however, be permissive of service potentially being removed VERY quickly:
services = get_service_list()
assert len(services) in (1, 2)
for service in services:
name = service['service']
assert name in (svc1, svc2)
if name == svc1:
assert service['yaml'] == 'svc'
else:
assert service['yaml'] == 'simple'
# svc2 should be getting uninstalled, svc1 shouldn't:
assert service['uninstall'] == (name == svc2)
# restart app and wait for removal to succeed after restart:
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_marathon.wait_for_app_running(config.SERVICE_NAME, sdk_marathon.TIMEOUT_SECONDS)
wait_for_service_count(1)
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'COMPLETE', multiservice_name=svc1)
# verify that svc.yml is still deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == ['hello-0:[server]', 'world-0:[server]', 'world-1:[server]']
def test_canary_init():
def fn():
return sdk_cmd.run_cli('hello-world pod list')
assert json.loads(shakedown.wait_for(fn, noisy=True)) == []
pl = sdk_plan.wait_for_plan_status(config.PACKAGE_NAME, 'deploy',
'WAITING')
log.info(pl)
assert pl['status'] == 'WAITING'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'WAITING'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
phase = pl['phases'][1]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'WAITING'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
def test_canary_init():
@retrying.retry(wait_fixed=1000, stop_max_delay=600 * 1000, retry_on_result=lambda res: not res)
def wait_for_empty():
# check for empty list internally rather than returning empty list.
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "pod list")
assert rc == 0, "Pod list failed"
return json.loads(stdout) == []
wait_for_empty()
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "deploy", "WAITING")
log.info(pl)
assert pl["status"] == "WAITING"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "WAITING"
assert steps[1]["status"] == "WAITING"
assert steps[2]["status"] == "PENDING"
assert steps[3]["status"] == "PENDING"
phase = pl["phases"][1]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "WAITING"
assert steps[1]["status"] == "WAITING"
assert steps[2]["status"] == "PENDING"
assert steps[3]["status"] == "PENDING"
def test_canary_init():
def fn():
# check for empty list internally rather than returning empty list.
# otherwise shakedown.wait_for() will keep going...
return sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'pod list', json=True) == []
assert shakedown.wait_for(fn, noisy=True, timeout_seconds=10 * 60)
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'WAITING')
log.info(pl)
assert pl['status'] == 'WAITING'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'WAITING'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
phase = pl['phases'][1]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'WAITING'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
def test_recovery_backoff():
sdk_install.install(
config.PACKAGE_NAME,
foldered_name,
expected_running_tasks=0,
additional_options=back_off_crash_loop_options,
wait_for_deployment=False,
wait_for_all_conditions=False,
)
check_delayed_and_suppressed("deploy")
sdk_plan.force_complete_step(foldered_name, "deploy", "crash",
"hello-0:[server]")
# Deploy plan is complete. Recovery plan should take over. Recovery plan is in COMPLETE
# by default, it should go from STARTED -> DELAYED.
sdk_plan.wait_for_plan_status(foldered_name, "recovery", "STARTED")
check_delayed_and_suppressed("recovery")
sdk_plan.wait_for_plan_status(foldered_name, "recovery", "STARTED")
def test_canary_init():
@retrying.retry(wait_fixed=1000,
stop_max_delay=600 * 1000,
retry_on_result=lambda res: not res)
def wait_for_empty():
# check for empty list internally rather than returning empty list.
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME,
config.SERVICE_NAME, "pod list")
assert rc == 0, "Pod list failed"
return json.loads(stdout) == []
wait_for_empty()
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "deploy",
"WAITING")
log.info(pl)
assert pl["status"] == "WAITING"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "WAITING"
assert steps[1]["status"] == "WAITING"
assert steps[2]["status"] == "PENDING"
assert steps[3]["status"] == "PENDING"
phase = pl["phases"][1]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "WAITING"
assert steps[1]["status"] == "WAITING"
assert steps[2]["status"] == "PENDING"
assert steps[3]["status"] == "PENDING"
def test_canary_init():
def fn():
# check for empty list internally rather than returning empty list.
# otherwise shakedown.wait_for() will keep going...
return sdk_cmd.svc_cli(config.PACKAGE_NAME,
config.SERVICE_NAME,
'pod list',
json=True) == []
assert shakedown.wait_for(fn, noisy=True, timeout_seconds=10 * 60)
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy',
'WAITING')
log.info(pl)
assert pl['status'] == 'WAITING'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'WAITING'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
phase = pl['phases'][1]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'WAITING'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
def test_increase_cpu():
hello_0_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-0-server')
sdk_marathon.bump_cpu_count_config(config.SERVICE_NAME, 'HELLO_CPUS')
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'WAITING')
log.info(pl)
assert pl['status'] == 'WAITING'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'WAITING'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
assert steps[4]['status'] == 'PENDING'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
# check that all prior tasks are still running, no changes yet
expected_tasks = [
'hello-0', 'hello-1', 'hello-2', 'hello-3', 'hello-4',
'world-0', 'world-1', 'world-2', 'world-3']
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'pod list', json=True) == expected_tasks
assert hello_0_ids == sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-0-server')
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'plan continue deploy hello-deploy')
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'hello-0-server', hello_0_ids)
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
pl = sdk_plan.wait_for_step_status(config.SERVICE_NAME, 'deploy', 'hello-deploy', 'hello-0:[server]', 'COMPLETE')
log.info(pl)
assert pl['status'] == 'WAITING'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
assert steps[4]['status'] == 'PENDING'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
hello_1_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-1-server')
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'plan continue deploy hello-deploy')
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'hello-1-server', hello_1_ids)
pl = sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
log.info(pl)
assert pl['status'] == 'COMPLETE'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
assert steps[4]['status'] == 'COMPLETE'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
def test_increase_count():
sdk_marathon.bump_task_count_config(config.SERVICE_NAME, 'HELLO_COUNT')
expected_tasks = [
'hello-0', 'hello-1', 'hello-2', 'hello-3', 'world-0', 'world-1',
'world-2', 'world-3'
]
try:
sdk_tasks.check_running(config.SERVICE_NAME,
len(expected_tasks) + 1,
timeout_seconds=60)
assert False, "Should not start task now"
except AssertionError as arg:
raise arg
except:
pass # expected to fail
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME,
config.SERVICE_NAME,
'pod list',
json=True) == expected_tasks
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy',
'WAITING')
log.info(pl)
assert pl['status'] == 'WAITING'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
assert steps[4]['status'] == 'WAITING'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
'plan continue deploy hello-deploy')
expected_tasks = [
'hello-0', 'hello-1', 'hello-2', 'hello-3', 'hello-4', 'world-0',
'world-1', 'world-2', 'world-3'
]
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME,
config.SERVICE_NAME,
'pod list',
json=True) == expected_tasks
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy',
'COMPLETE')
log.info(pl)
assert pl['status'] == 'COMPLETE'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
assert steps[4]['status'] == 'COMPLETE'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
def test_increase_cpu():
hello_0_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-0-server')
config.bump_hello_cpus()
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy',
'WAITING')
log.info(pl)
assert pl['status'] == 'WAITING'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'WAITING'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
assert steps[4]['status'] == 'PENDING'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
# check that all prior tasks are still running, no changes yet
expected_tasks = [
'hello-0', 'hello-1', 'hello-2', 'hello-3', 'hello-4', 'world-0',
'world-1', 'world-2', 'world-3'
]
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME,
config.SERVICE_NAME,
'pod list',
json=True) == expected_tasks
assert hello_0_ids == sdk_tasks.get_task_ids(config.SERVICE_NAME,
'hello-0-server')
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
'plan continue deploy hello-deploy')
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'hello-0-server',
hello_0_ids)
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
pl = sdk_plan.wait_for_step_status(config.SERVICE_NAME, 'deploy',
'hello-deploy', 'hello-0:[server]',
'COMPLETE')
log.info(pl)
assert pl['status'] == 'WAITING'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'WAITING'
assert steps[2]['status'] == 'PENDING'
assert steps[3]['status'] == 'PENDING'
assert steps[4]['status'] == 'PENDING'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
hello_1_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-1-server')
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
'plan continue deploy hello-deploy')
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'hello-1-server',
hello_1_ids)
pl = sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
log.info(pl)
assert pl['status'] == 'COMPLETE'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
assert steps[4]['status'] == 'COMPLETE'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
def check_healthy(expected_recovery_state="COMPLETE"):
config.check_running()
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "recovery",
expected_recovery_state)
def test_region_config_update_does_not_succeed(configure_universe, local_service):
change_region_config(REMOTE_REGION)
sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'ERROR', timeout_seconds=180)
change_region_config(None)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME, timeout_seconds=180)
def test_increase_cpu():
hello_0_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, "hello-0-server")
config.bump_hello_cpus()
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "deploy", "WAITING")
log.info(pl)
assert pl["status"] == "WAITING"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "WAITING"
assert steps[1]["status"] == "WAITING"
assert steps[2]["status"] == "PENDING"
assert steps[3]["status"] == "PENDING"
assert steps[4]["status"] == "PENDING"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
# check that all prior tasks are still running, no changes yet
expected_tasks = [
"hello-0",
"hello-1",
"hello-2",
"hello-3",
"hello-4",
"world-0",
"world-1",
"world-2",
"world-3",
]
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "pod list")
assert rc == 0, "Pod list failed"
assert json.loads(stdout) == expected_tasks
assert hello_0_ids == sdk_tasks.get_task_ids(config.SERVICE_NAME, "hello-0-server")
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "plan continue deploy hello-deploy")
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, "hello-0-server", hello_0_ids)
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
pl = sdk_plan.wait_for_step_status(
config.SERVICE_NAME, "deploy", "hello-deploy", "hello-0:[server]", "COMPLETE"
)
log.info(pl)
assert pl["status"] == "WAITING"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "WAITING"
assert steps[2]["status"] == "PENDING"
assert steps[3]["status"] == "PENDING"
assert steps[4]["status"] == "PENDING"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
hello_1_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, "hello-1-server")
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "plan continue deploy hello-deploy")
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, "hello-1-server", hello_1_ids)
pl = sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
log.info(pl)
assert pl["status"] == "COMPLETE"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
assert steps[4]["status"] == "COMPLETE"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
def test_increase_cpu():
hello_0_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, "hello-0-server")
config.bump_hello_cpus()
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "deploy",
"WAITING")
log.info(pl)
assert pl["status"] == "WAITING"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "WAITING"
assert steps[1]["status"] == "WAITING"
assert steps[2]["status"] == "PENDING"
assert steps[3]["status"] == "PENDING"
assert steps[4]["status"] == "PENDING"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
# check that all prior tasks are still running, no changes yet
expected_tasks = [
"hello-0",
"hello-1",
"hello-2",
"hello-3",
"hello-4",
"world-0",
"world-1",
"world-2",
"world-3",
]
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"pod list")
assert rc == 0, "Pod list failed"
assert json.loads(stdout) == expected_tasks
assert hello_0_ids == sdk_tasks.get_task_ids(config.SERVICE_NAME,
"hello-0-server")
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"plan continue deploy hello-deploy")
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, "hello-0-server",
hello_0_ids)
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
pl = sdk_plan.wait_for_step_status(config.SERVICE_NAME, "deploy",
"hello-deploy", "hello-0:[server]",
"COMPLETE")
log.info(pl)
assert pl["status"] == "WAITING"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "WAITING"
assert steps[2]["status"] == "PENDING"
assert steps[3]["status"] == "PENDING"
assert steps[4]["status"] == "PENDING"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
hello_1_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, "hello-1-server")
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"plan continue deploy hello-deploy")
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, "hello-1-server",
hello_1_ids)
pl = sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
log.info(pl)
assert pl["status"] == "COMPLETE"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
assert steps[4]["status"] == "COMPLETE"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
def test_increase_count():
sdk_marathon.bump_task_count_config(config.SERVICE_NAME, "HELLO_COUNT")
expected_tasks = [
"hello-0",
"hello-1",
"hello-2",
"hello-3",
"world-0",
"world-1",
"world-2",
"world-3",
]
try:
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks) + 1, timeout_seconds=60)
assert False, "Should not start task now"
except AssertionError as arg:
raise arg
except Exception:
pass # expected to fail
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "pod list")
assert rc == 0, "Pod list failed"
assert json.loads(stdout) == expected_tasks
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "deploy", "WAITING")
log.info(pl)
assert pl["status"] == "WAITING"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
assert steps[4]["status"] == "WAITING"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "plan continue deploy hello-deploy")
expected_tasks = [
"hello-0",
"hello-1",
"hello-2",
"hello-3",
"hello-4",
"world-0",
"world-1",
"world-2",
"world-3",
]
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, "pod list")
assert rc == 0, "Pod list failed"
assert json.loads(stdout) == expected_tasks
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "deploy", "COMPLETE")
log.info(pl)
assert pl["status"] == "COMPLETE"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
assert steps[4]["status"] == "COMPLETE"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
def check_healthy(expected_recovery_state="COMPLETE"):
config.check_running()
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "recovery", expected_recovery_state)
def test_add_multiple_uninstall():
# add two services:
svc1 = "test1"
sdk_cmd.service_request("POST",
config.SERVICE_NAME,
"/v1/multi/{}?yaml=svc".format(svc1),
json=service_params(svc1))
svc2 = "test2"
sdk_cmd.service_request(
"POST",
config.SERVICE_NAME,
"/v1/multi/{}?yaml=simple".format(svc2),
json=service_params(svc2),
)
# get list, should immediately have new entries:
services = get_service_list()
assert len(services) == 2
for service in services:
name = service["service"]
assert name in (svc1, svc2)
if name == svc1:
assert service["yaml"] == "svc"
else:
assert service["yaml"] == "simple"
assert not service["uninstall"]
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc1)
# verify that svc.yml was deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == [
"hello-0:[server]",
"world-0:[server]",
"world-1:[server]",
]
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc2)
# verify that simple.yml was deployed as svc2:
assert sdk_plan.get_all_step_names(plan) == ["hello-0:[server]"]
# remove one service, then immediately restart app to verify recovery during service removal:
sdk_cmd.service_request("DELETE", config.SERVICE_NAME,
"/v1/multi/{}".format(svc2))
# check delete bits is set. however, be permissive of service potentially being removed VERY quickly:
services = get_service_list()
assert len(services) in (1, 2)
for service in services:
name = service["service"]
assert name in (svc1, svc2)
if name == svc1:
assert service["yaml"] == "svc"
else:
assert service["yaml"] == "simple"
# svc2 should be getting uninstalled, svc1 shouldn't:
assert service["uninstall"] == (name == svc2)
# restart app and wait for removal to succeed after restart:
sdk_marathon.restart_app(config.SERVICE_NAME)
wait_for_service_count(1)
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc1)
# verify that svc.yml is still deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == [
"hello-0:[server]",
"world-0:[server]",
"world-1:[server]",
]
def test_increase_count():
sdk_marathon.bump_task_count_config(config.SERVICE_NAME, 'HELLO_COUNT')
expected_tasks = [
'hello-0', 'hello-1', 'hello-2', 'hello-3',
'world-0', 'world-1', 'world-2', 'world-3']
try:
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks) + 1, timeout_seconds=60)
assert False, "Should not start task now"
except AssertionError as arg:
raise arg
except:
pass # expected to fail
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'pod list', json=True) == expected_tasks
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'WAITING')
log.info(pl)
assert pl['status'] == 'WAITING'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'WAITING'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
assert steps[4]['status'] == 'WAITING'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'plan continue deploy hello-deploy')
expected_tasks = [
'hello-0', 'hello-1', 'hello-2', 'hello-3', 'hello-4',
'world-0', 'world-1', 'world-2', 'world-3']
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
assert sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME, 'pod list', json=True) == expected_tasks
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'COMPLETE')
log.info(pl)
assert pl['status'] == 'COMPLETE'
assert len(pl['phases']) == 2
phase = pl['phases'][0]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 5
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
assert steps[4]['status'] == 'COMPLETE'
phase = pl['phases'][1]
assert phase['status'] == 'COMPLETE'
steps = phase['steps']
assert len(steps) == 4
assert steps[0]['status'] == 'COMPLETE'
assert steps[1]['status'] == 'COMPLETE'
assert steps[2]['status'] == 'COMPLETE'
assert steps[3]['status'] == 'COMPLETE'
def test_increase_count():
sdk_marathon.bump_task_count_config(config.SERVICE_NAME, "HELLO_COUNT")
expected_tasks = [
"hello-0",
"hello-1",
"hello-2",
"hello-3",
"world-0",
"world-1",
"world-2",
"world-3",
]
try:
sdk_tasks.check_running(config.SERVICE_NAME,
len(expected_tasks) + 1,
timeout_seconds=60)
assert False, "Should not start task now"
except AssertionError as arg:
raise arg
except Exception:
pass # expected to fail
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"pod list")
assert rc == 0, "Pod list failed"
assert json.loads(stdout) == expected_tasks
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "deploy",
"WAITING")
log.info(pl)
assert pl["status"] == "WAITING"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "WAITING"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
assert steps[4]["status"] == "WAITING"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"plan continue deploy hello-deploy")
expected_tasks = [
"hello-0",
"hello-1",
"hello-2",
"hello-3",
"hello-4",
"world-0",
"world-1",
"world-2",
"world-3",
]
sdk_tasks.check_running(config.SERVICE_NAME, len(expected_tasks))
rc, stdout, _ = sdk_cmd.svc_cli(config.PACKAGE_NAME, config.SERVICE_NAME,
"pod list")
assert rc == 0, "Pod list failed"
assert json.loads(stdout) == expected_tasks
pl = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, "deploy",
"COMPLETE")
log.info(pl)
assert pl["status"] == "COMPLETE"
assert len(pl["phases"]) == 2
phase = pl["phases"][0]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 5
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
assert steps[4]["status"] == "COMPLETE"
phase = pl["phases"][1]
assert phase["status"] == "COMPLETE"
steps = phase["steps"]
assert len(steps) == 4
assert steps[0]["status"] == "COMPLETE"
assert steps[1]["status"] == "COMPLETE"
assert steps[2]["status"] == "COMPLETE"
assert steps[3]["status"] == "COMPLETE"
