def test_pod_health_failed_check():
"""Deploys a pod with correct health checks, then partitions the network and verifies that
the tasks get restarted with new task IDs.
"""
pod_def = pods.ports_pod()
pod_id = pod_def['id']
host = common.ip_other_than_mom()
common.pin_pod_to_host(pod_def, host)
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
initial_id1 = tasks[0]['id']
initial_id2 = tasks[1]['id']
pod = client.list_pod()[0]
container1 = pod['instances'][0]['containers'][0]
port = container1['endpoints'][0]['allocatedHostPort']
common.block_iptable_rules_for_seconds(host,
port,
7,
block_input=True,
block_output=False)
common.deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
for new_task in tasks:
new_task_id = new_task['id']
assert new_task_id != initial_id1, f"Task {new_task_id} has not been restarted" # NOQA E999
assert new_task_id != initial_id2, f"Task {new_task_id} has not been restarted"
def test_marathon_when_disconnected_from_zk():
"""Launches an app from Marathon, then knocks out access to ZK from Marathon.
Verifies the task is preserved.
"""
app_def = apps.sleep_app()
app_id = app_def["id"]
host = common.ip_other_than_mom()
common.pin_to_host(app_def, host)
client = marathon.create_client()
client.add_app(app_def)
deployment_wait(service_id=app_id)
tasks = client.get_tasks(app_id)
original_task_id = tasks[0]['id']
common.block_iptable_rules_for_seconds(host, 2181, sleep_seconds=10, block_input=True, block_output=False)
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_task_is_back():
tasks = client.get_tasks(app_id)
assert tasks[0]['id'] == original_task_id, \
"The task {} got replaced with {}".format(original_task_id, tasks[0]['id'])
check_task_is_back()
def test_marathon_when_disconnected_from_zk():
"""Launches an app from Marathon, then knocks out access to ZK from Marathon.
Verifies the task is preserved.
"""
app_def = apps.sleep_app()
host = common.ip_other_than_mom()
common.pin_to_host(app_def, host)
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait()
tasks = client.get_tasks(app_def["id"])
original_task_id = tasks[0]['id']
common.block_iptable_rules_for_seconds(host,
2181,
sleep_seconds=10,
block_input=True,
block_output=False)
@retrying.retry(wait_fixed=1000,
stop_max_attempt_number=30,
retry_on_exception=common.ignore_exception)
def check_task_is_back():
tasks = client.get_tasks(app_def["id"])
assert tasks[0]['id'] == original_task_id, \
"The task {} got replaced with {}".format(original_task_id, tasks[0]['id'])
check_task_is_back()
def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
common.block_iptable_rules_for_seconds(original_leader, 2181, sleep_seconds=30)
common.assert_marathon_leadership_changed(original_leader)
def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
common.block_iptable_rules_for_seconds(original_leader, 2181, sleep_seconds=30)
common.assert_marathon_leadership_changed(original_leader)
def test_pod_health_failed_check():
"""Deploys a pod with correct health checks, then partitions the network and verifies that
the tasks get restarted with new task IDs.
"""
pod_def = pods.ports_pod()
pod_id = pod_def['id']
host = common.ip_other_than_mom()
common.pin_pod_to_host(pod_def, host)
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
initial_id1 = tasks[0]['id']
initial_id2 = tasks[1]['id']
pod = client.list_pod()[0]
container1 = pod['instances'][0]['containers'][0]
port = container1['endpoints'][0]['allocatedHostPort']
common.block_iptable_rules_for_seconds(host, port, 7, block_input=True, block_output=False)
common.deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
for new_task in tasks:
new_task_id = new_task['id']
assert new_task_id != initial_id1, f"Task {new_task_id} has not been restarted" # NOQA E999
assert new_task_id != initial_id2, f"Task {new_task_id} has not been restarted"
def test_task_gets_restarted_due_to_network_split():
"""Verifies that a health check fails in presence of a network partition."""
app_def = apps.http_server()
app_id = app_def["id"]
app_def['healthChecks'] = [common.health_check()]
common.pin_to_host(app_def, common.ip_other_than_mom())
client = marathon.create_client()
client.add_app(app_def)
common.deployment_wait(service_id=app_id)
app = client.get_app(app_id)
assert app['tasksRunning'] == 1, \
"The number of running tasks is {}, but 1 was expected".format(app['tasksRunning'])
assert app['tasksHealthy'] == 1, \
"The number of healthy tasks is {}, but 1 was expected".format(app['tasksHealthy'])
tasks = client.get_tasks(app_id)
task_id = tasks[0]['id']
host = tasks[0]['host']
port = tasks[0]['ports'][0]
# introduce a network partition
common.block_iptable_rules_for_seconds(host,
port,
sleep_seconds=10,
block_input=True,
block_output=False)
common.deployment_wait(service_id=app_id)
app = client.get_app(app_id)
tasks = client.get_tasks(app_id)
new_task_id = tasks[0]['id']
assert task_id != new_task_id, "The task didn't get killed because of a failed health check"
assert app['tasksRunning'] == 1, \
"The number of running tasks is {}, but 1 was expected".format(app['tasksRunning'])
assert app['tasksHealthy'] == 1, \
"The number of healthy tasks is {}, but 0 was expected".format(app['tasksHealthy'])
# network partition should cause a task restart
@retrying.retry(wait_fixed=1000,
stop_max_attempt_number=30,
retry_on_exception=common.ignore_exception)
def check_health_message():
tasks = client.get_tasks(app_id)
new_task_id = tasks[0]['id']
assert task_id != new_task_id, "The task has not been restarted: {}".format(
task_id)
app = client.get_app(app_id)
assert app['tasksRunning'] == 1, \
"The number of running tasks is {}, but 1 was expected".format(app['tasksRunning'])
assert app['tasksHealthy'] == 1, \
"The number of healthy tasks is {}, but 1 was expected".format(app['tasksHealthy'])
check_health_message()
def test_marathon_master_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking outbound connection to mesos master
# Marathon has a Mesos heartbeat interval of 15 seconds. If 5 are missed it
# disconnects. Thus we should wait more than 75 seconds.
common.block_iptable_rules_for_seconds(original_leader, 5050, sleep_seconds=100,
block_input=False, block_output=True)
common.assert_marathon_leadership_changed(original_leader)
def test_marathon_master_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking outbound connection to mesos master
# Marathon has a Mesos heartbeat interval of 15 seconds. If 5 are missed it
# disconnects. Thus we should wait more than 75 seconds.
common.block_iptable_rules_for_seconds(original_leader, 5050, sleep_seconds=100,
block_input=False, block_output=True)
common.assert_marathon_leadership_changed(original_leader)
def test_marathon_zk_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
common.block_iptable_rules_for_seconds(original_leader,
2181,
sleep_seconds=30)
common.marathon_leadership_changed(original_leader)
# Make sure marathon is available
shakedown.wait_for_service_endpoint(marathon_service_name,
timedelta(minutes=5).total_seconds())
def test_task_gets_restarted_due_to_network_split():
"""Verifies that a health check fails in presence of a network partition."""
app_def = apps.http_server()
app_def['healthChecks'] = [common.health_check()]
common.pin_to_host(app_def, common.ip_other_than_mom())
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait()
app = client.get_app(app_def["id"])
assert app['tasksRunning'] == 1, \
"The number of running tasks is {}, but 1 was expected".format(app['tasksRunning'])
assert app['tasksHealthy'] == 1, \
"The number of healthy tasks is {}, but 1 was expected".format(app['tasksHealthy'])
tasks = client.get_tasks(app_def["id"])
task_id = tasks[0]['id']
host = tasks[0]['host']
port = tasks[0]['ports'][0]
# introduce a network partition
common.block_iptable_rules_for_seconds(host, port, sleep_seconds=10, block_input=True, block_output=False)
shakedown.deployment_wait()
app = client.get_app(app_def["id"])
tasks = client.get_tasks(app_def["id"])
new_task_id = tasks[0]['id']
assert task_id != new_task_id, "The task didn't get killed because of a failed health check"
assert app['tasksRunning'] == 1, \
"The number of running tasks is {}, but 1 was expected".format(app['tasksRunning'])
assert app['tasksHealthy'] == 1, \
"The number of healthy tasks is {}, but 0 was expected".format(app['tasksHealthy'])
# network partition should cause a task restart
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_health_message():
tasks = client.get_tasks(app_def["id"])
new_task_id = tasks[0]['id']
assert task_id != new_task_id, "The task has not been restarted: {}".format(task_id)
app = client.get_app(app_def["id"])
assert app['tasksRunning'] == 1, \
"The number of running tasks is {}, but 1 was expected".format(app['tasksRunning'])
assert app['tasksHealthy'] == 1, \
"The number of healthy tasks is {}, but 1 was expected".format(app['tasksHealthy'])
check_health_message()
def test_marathon_master_partition_leader_change(marathon_service_name):
original_leader = common.get_marathon_leader_not_on_master_leader_node()
# blocking outbound connection to mesos master
common.block_iptable_rules_for_seconds(original_leader,
5050,
sleep_seconds=60,
block_input=False,
block_output=True)
common.marathon_leadership_changed(original_leader)
# Make sure marathon is available
shakedown.wait_for_service_endpoint(marathon_service_name,
timedelta(minutes=5).total_seconds())
def test_task_gets_restarted_due_to_network_split():
"""Verifies that a health check fails in presence of a network partition."""
app_def = apps.http_server("app-network-split")
app_id = app_def["id"]
app_def['healthChecks'] = [common.health_check()]
common.pin_to_host(app_def, common.ip_other_than_mom())
client = marathon.create_client()
client.add_app(app_def)
deployment_wait(service_id=app_id)
app = client.get_app(app_id)
assert app['tasksRunning'] == 1, \
"The number of running tasks is {}, but 1 was expected".format(app['tasksRunning'])
assert app['tasksHealthy'] == 1, \
"The number of healthy tasks is {}, but 1 was expected".format(app['tasksHealthy'])
tasks = client.get_tasks(app_id)
task_id = tasks[0]['id']
host = tasks[0]['host']
port = tasks[0]['ports'][0]
# introduce a network partition
common.block_iptable_rules_for_seconds(host, port, sleep_seconds=10, block_input=True, block_output=False)
# Network partition should cause the task to restart N times until the partition is resolved (since we
# pinned the task to the split agent). A new task with a new taskId should eventually be running and healthy.
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_health_message():
tasks = client.get_tasks(app_id)
new_task_id = tasks[0]['id']
assert task_id != new_task_id, "The task has not been restarted: {}".format(task_id)
app = client.get_app(app_id)
assert app['tasksRunning'] == 1, \
"The number of running tasks is {}, but 1 was expected".format(app['tasksRunning'])
assert app['tasksHealthy'] == 1, \
"The number of healthy tasks is {}, but 1 was expected".format(app['tasksHealthy'])
check_health_message()
