def test_add_deploy_restart_remove():
svc1 = 'test1'
# add svc as test1:
sdk_cmd.service_request('POST',
config.SERVICE_NAME,
'/v1/multi/{}?yaml=svc'.format(svc1),
json=service_params(svc1))
# get list, should immediately have new entry:
service = get_service_list()[0]
assert service['service'] == svc1
assert service['yaml'] == 'svc'
assert not service['uninstall']
sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
'deploy',
'COMPLETE',
multiservice_name=svc1)
task_ids = sdk_tasks.get_task_ids('marathon', config.SERVICE_NAME)
log.info('list of task ids {}'.format(task_ids))
old_task_id = task_ids[0]
# restart and check that service is recovered:
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_marathon.wait_for_app_running(config.SERVICE_NAME,
sdk_marathon.TIMEOUT_SECONDS)
# check that scheduler task was relaunched
check_scheduler_relaunched(config.SERVICE_NAME, old_task_id)
service = wait_for_service_count(1)[0]
assert service['service'] == svc1
assert service['yaml'] == 'svc'
assert not service['uninstall']
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
'deploy',
'COMPLETE',
multiservice_name=svc1)
# verify that svc.yml was deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == [
'hello-0:[server]', 'world-0:[server]', 'world-1:[server]'
]
# trigger service removal, wait for removal:
sdk_cmd.service_request('DELETE', config.SERVICE_NAME,
'/v1/multi/{}'.format(svc1))
# check delete bit is set. however, be permissive of service being removed VERY quickly:
services = get_service_list()
assert len(services) <= 1
for service in services:
assert service['service'] == svc1
assert service['yaml'] == 'svc'
assert service['uninstall']
wait_for_service_count(0)
def test_add_deploy_restart_remove():
svc1 = "test1"
# add svc as test1:
sdk_cmd.service_request("POST",
config.SERVICE_NAME,
"/v1/multi/{}?yaml=svc".format(svc1),
json=service_params(svc1))
# get list, should immediately have new entry:
service = get_service_list()[0]
assert service["service"] == svc1
assert service["yaml"] == "svc"
assert not service["uninstall"]
sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc1)
task_ids = sdk_tasks.get_task_ids("marathon", config.SERVICE_NAME)
log.info("list of task ids {}".format(task_ids))
old_task_id = task_ids[0]
# restart and check that service is recovered:
sdk_marathon.restart_app(config.SERVICE_NAME)
# check that scheduler task was relaunched
sdk_tasks.check_scheduler_relaunched(config.SERVICE_NAME, old_task_id)
service = wait_for_service_count(1)[0]
assert service["service"] == svc1
assert service["yaml"] == "svc"
assert not service["uninstall"]
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc1)
# verify that svc.yml was deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == [
"hello-0:[server]",
"world-0:[server]",
"world-1:[server]",
]
# trigger service removal, wait for removal:
sdk_cmd.service_request("DELETE", config.SERVICE_NAME,
"/v1/multi/{}".format(svc1))
# check delete bit is set. however, be permissive of service being removed VERY quickly:
services = get_service_list()
assert len(services) <= 1
for service in services:
assert service["service"] == svc1
assert service["yaml"] == "svc"
assert service["uninstall"]
wait_for_service_count(0)
def test_add_multiple_uninstall():
# add two services:
svc1 = 'test1'
sdk_cmd.service_request('POST', config.SERVICE_NAME, '/v1/multi/{}?yaml=svc'.format(svc1), json=service_params(svc1))
svc2 = 'test2'
sdk_cmd.service_request('POST', config.SERVICE_NAME, '/v1/multi/{}?yaml=simple'.format(svc2), json=service_params(svc2))
# get list, should immediately have new entries:
services = get_service_list()
assert len(services) == 2
for service in services:
name = service['service']
assert name in (svc1, svc2)
if name == svc1:
assert service['yaml'] == 'svc'
else:
assert service['yaml'] == 'simple'
assert not service['uninstall']
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'COMPLETE', multiservice_name=svc1)
# verify that svc.yml was deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == ['hello-0:[server]', 'world-0:[server]', 'world-1:[server]']
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'COMPLETE', multiservice_name=svc2)
# verify that simple.yml was deployed as svc2:
assert sdk_plan.get_all_step_names(plan) == ['hello-0:[server]']
# remove one service, then immediately restart app to verify recovery during service removal:
sdk_cmd.service_request('DELETE', config.SERVICE_NAME, '/v1/multi/{}'.format(svc2))
# check delete bits is set. however, be permissive of service potentially being removed VERY quickly:
services = get_service_list()
assert len(services) in (1, 2)
for service in services:
name = service['service']
assert name in (svc1, svc2)
if name == svc1:
assert service['yaml'] == 'svc'
else:
assert service['yaml'] == 'simple'
# svc2 should be getting uninstalled, svc1 shouldn't:
assert service['uninstall'] == (name == svc2)
# restart app and wait for removal to succeed after restart:
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_marathon.wait_for_app_running(config.SERVICE_NAME, sdk_marathon.TIMEOUT_SECONDS)
wait_for_service_count(1)
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME, 'deploy', 'COMPLETE', multiservice_name=svc1)
# verify that svc.yml is still deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == ['hello-0:[server]', 'world-0:[server]', 'world-1:[server]']
def test_toxic_sidecar_doesnt_trigger_recovery():
# 1. Run the toxic sidecar plan that will never succeed.
# 2. Restart the scheduler.
# 3. Verify that its recovery plan has not changed, as a failed ONCE task should
# never trigger recovery
initial_recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert (initial_recovery_plan['status'] == "COMPLETE")
log.info(initial_recovery_plan)
sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar-toxic')
wait_for_toxic_sidecar()
# Restart the scheduler and wait for it to come up.
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
# Now, verify that its recovery plan hasn't changed.
final_recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert (initial_recovery_plan['status'] == final_recovery_plan['status'])
def test_toxic_sidecar_doesnt_trigger_recovery():
# 1. Run the toxic sidecar plan that will never succeed.
# 2. Restart the scheduler.
# 3. Verify that its recovery plan has not changed, as a failed ONCE task should
# never trigger recovery
initial_recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert(initial_recovery_plan['status'] == "COMPLETE")
log.info(initial_recovery_plan)
sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar-toxic')
wait_for_toxic_sidecar()
# Restart the scheduler and wait for it to come up.
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
# Now, verify that its recovery plan hasn't changed.
final_recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert(initial_recovery_plan['status'] == final_recovery_plan['status'])
def test_toxic_sidecar_doesnt_trigger_recovery():
# 1. Run the toxic sidecar plan that will never succeed.
# 2. Restart the scheduler.
# 3. Verify that its recovery plan is empty, as a failed ONCE task should
# never trigger recovery
recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert (len(recovery_plan['phases']) == 0)
log.info(recovery_plan)
sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar-toxic')
wait_for_toxic_sidecar()
# Restart the scheduler and wait for it to come up.
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
# Now, verify that its recovery plan is empty.
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'recovery')
recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert (len(recovery_plan['phases']) == 0)
def test_toxic_sidecar_doesnt_trigger_recovery():
# 1. Run the toxic sidecar plan that will never succeed.
# 2. Restart the scheduler.
# 3. Verify that its recovery plan is empty, as a failed FINISHED task should
# never trigger recovery
recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert(len(recovery_plan['phases']) == 0)
log.info(recovery_plan)
sdk_plan.start_plan(config.SERVICE_NAME, 'sidecar-toxic')
shakedown.wait_for(ToxicSidecarCheck(), timeout_seconds=10 * 60)
# Restart the scheduler and wait for it to come up.
sdk_marathon.restart_app(config.SERVICE_NAME)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
# Now, verify that its recovery plan is empty.
sdk_plan.wait_for_completed_plan(config.SERVICE_NAME, 'recovery')
recovery_plan = sdk_plan.get_plan(config.SERVICE_NAME, 'recovery')
assert(len(recovery_plan['phases']) == 0)
def test_add_multiple_uninstall():
# add two services:
svc1 = "test1"
sdk_cmd.service_request("POST",
config.SERVICE_NAME,
"/v1/multi/{}?yaml=svc".format(svc1),
json=service_params(svc1))
svc2 = "test2"
sdk_cmd.service_request(
"POST",
config.SERVICE_NAME,
"/v1/multi/{}?yaml=simple".format(svc2),
json=service_params(svc2),
)
# get list, should immediately have new entries:
services = get_service_list()
assert len(services) == 2
for service in services:
name = service["service"]
assert name in (svc1, svc2)
if name == svc1:
assert service["yaml"] == "svc"
else:
assert service["yaml"] == "simple"
assert not service["uninstall"]
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc1)
# verify that svc.yml was deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == [
"hello-0:[server]",
"world-0:[server]",
"world-1:[server]",
]
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc2)
# verify that simple.yml was deployed as svc2:
assert sdk_plan.get_all_step_names(plan) == ["hello-0:[server]"]
# remove one service, then immediately restart app to verify recovery during service removal:
sdk_cmd.service_request("DELETE", config.SERVICE_NAME,
"/v1/multi/{}".format(svc2))
# check delete bits is set. however, be permissive of service potentially being removed VERY quickly:
services = get_service_list()
assert len(services) in (1, 2)
for service in services:
name = service["service"]
assert name in (svc1, svc2)
if name == svc1:
assert service["yaml"] == "svc"
else:
assert service["yaml"] == "simple"
# svc2 should be getting uninstalled, svc1 shouldn't:
assert service["uninstall"] == (name == svc2)
# restart app and wait for removal to succeed after restart:
sdk_marathon.restart_app(config.SERVICE_NAME)
wait_for_service_count(1)
plan = sdk_plan.wait_for_plan_status(config.SERVICE_NAME,
"deploy",
"COMPLETE",
multiservice_name=svc1)
# verify that svc.yml is still deployed as svc1:
assert sdk_plan.get_all_step_names(plan) == [
"hello-0:[server]",
"world-0:[server]",
"world-1:[server]",
]