def test_pinned_task_recovers_on_host():
"""Tests that when a pinned task gets killed, it recovers on the node it was pinned to."""
app_def = apps.sleep_app()
host = common.ip_other_than_mom()
common.pin_to_host(app_def, host)
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait()
tasks = client.get_tasks(app_def["id"])
common.kill_process_on_host(host, '[s]leep')
shakedown.deployment_wait()
@retrying.retry(wait_fixed=1000,
stop_max_attempt_number=30,
retry_on_exception=common.ignore_exception)
def check_for_new_task():
new_tasks = client.get_tasks(app_def["id"])
assert tasks[0]['id'] != new_tasks[0][
'id'], "The task did not get killed: {}".format(tasks[0]['id'])
assert new_tasks[0]['host'] == host, \
"The task got restarted on {}, but it was supposed to stay on {}".format(new_tasks[0]['host'], host)
check_for_new_task()
def test_pinned_task_recovers_on_host():
"""Tests that when a pinned task gets killed, it recovers on the node it was pinned to."""
app_def = apps.sleep_app()
app_id = app_def["id"]
host = common.ip_other_than_mom()
common.pin_to_host(app_def, host)
client = marathon.create_client()
client.add_app(app_def)
deployment_wait(service_id=app_id)
tasks = client.get_tasks(app_id)
common.kill_process_on_host(host, '[s]leep')
deployment_wait(service_id=app_id)
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_for_new_task():
new_tasks = client.get_tasks(app_id)
assert tasks[0]['id'] != new_tasks[0]['id'], "The task did not get killed: {}".format(tasks[0]['id'])
assert new_tasks[0]['host'] == host, \
"The task got restarted on {}, but it was supposed to stay on {}".format(new_tasks[0]['host'], host)
check_for_new_task()
def test_mom_when_mom_process_killed():
"""Launched a task from MoM then killed MoM."""
app_def = apps.sleep_app()
app_id = app_def["id"]
host = common.ip_other_than_mom()
common.pin_to_host(app_def, host)
with shakedown.marathon_on_marathon():
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait()
tasks = client.get_tasks(app_id)
original_task_id = tasks[0]['id']
common.kill_process_on_host(common.ip_of_mom(), 'marathon-assembly')
shakedown.wait_for_task('marathon', 'marathon-user', 300)
common.wait_for_service_endpoint('marathon-user', path="ping")
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_task_is_back():
tasks = client.get_tasks(app_id)
assert tasks[0]['id'] == original_task_id, "The task ID has changed"
check_task_is_back()
def test_task_failure_recovers():
"""Tests that if a task is KILLED, another one will be launched with a different ID."""
app_def = apps.sleep_app()
app_def['cmd'] = 'sleep 1000'
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait(app_id=app_def["id"])
tasks = client.get_tasks(app_def["id"])
old_task_id = tasks[0]['id']
host = tasks[0]['host']
common.kill_process_on_host(host, '[s]leep 1000')
shakedown.deployment_wait()
@retrying.retry(wait_fixed=1000,
stop_max_attempt_number=30,
retry_on_exception=common.ignore_exception)
def check_new_task_id():
tasks = client.get_tasks(app_def["id"])
new_task_id = tasks[0]['id']
assert old_task_id != new_task_id, "The task ID has not changed: {}".format(
old_task_id)
check_new_task_id()
def test_task_failure_recovers():
"""Tests that if a task is KILLED, another one will be launched with a different ID."""
app_def = apps.sleep_app()
app_def['cmd'] = 'sleep 1000'
app_id = app_def["id"]
client = marathon.create_client()
client.add_app(app_def)
deployment_wait(service_id=app_id)
tasks = client.get_tasks(app_id)
old_task_id = tasks[0]['id']
host = tasks[0]['host']
common.kill_process_on_host(host, '[s]leep 1000')
assert_that(lambda: client.get_tasks(app_id)[0],
eventually(has_value('id', not_(equal_to(old_task_id))), max_attempts=30))
def test_task_failure_recovers():
"""Tests that if a task is KILLED, another one will be launched with a different ID."""
app_def = apps.sleep_app()
app_def['cmd'] = 'sleep 1000'
app_id = app_def["id"]
client = marathon.create_client()
client.add_app(app_def)
common.deployment_wait(service_id=app_id)
tasks = client.get_tasks(app_id)
old_task_id = tasks[0]['id']
host = tasks[0]['host']
common.kill_process_on_host(host, '[s]leep 1000')
assert_that(lambda: client.get_tasks(app_id)[0],
eventually(has_value('id', not_(equal_to(old_task_id))), max_attempts=30))
def test_task_failure_recovers():
"""Tests that if a task is KILLED, another one will be launched with a different ID."""
app_def = apps.sleep_app()
app_def['cmd'] = 'sleep 1000'
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait(app_id=app_def["id"])
tasks = client.get_tasks(app_def["id"])
old_task_id = tasks[0]['id']
host = tasks[0]['host']
common.kill_process_on_host(host, '[s]leep 1000')
shakedown.deployment_wait()
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_new_task_id():
tasks = client.get_tasks(app_def["id"])
new_task_id = tasks[0]['id']
assert old_task_id != new_task_id, "The task ID has not changed: {}".format(old_task_id)
check_new_task_id()
def test_mom_when_mom_process_killed():
"""Launched a task from MoM then killed MoM."""
app_def = apps.sleep_app()
app_id = app_def["id"]
host = common.ip_other_than_mom()
common.pin_to_host(app_def, host)
with marathon_on_marathon() as client:
client.add_app(app_def)
deployment_wait(service_id=app_id, client=client)
tasks = client.get_tasks(app_id)
original_task_id = tasks[0]['id']
common.kill_process_on_host(common.ip_of_mom(), 'marathon-assembly')
wait_for_task('marathon', 'marathon-user', 300)
wait_for_service_endpoint('marathon-user', path="ping")
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_task_is_back():
tasks = client.get_tasks(app_id)
assert tasks[0]['id'] == original_task_id, "The task ID has changed"
check_task_is_back()
def kill_task(host, pattern):
pids = common.kill_process_on_host(host, pattern)
assert len(pids) != 0, "no task got killed on {} for pattern {}".format(host, pattern)
def kill_task(host, pattern):
pids = common.kill_process_on_host(host, pattern)
assert len(
pids) != 0, "no task got killed on {} for pattern {}".format(
host, pattern)
