def test_config_update_then_scheduler_died():
world_ids = sdk_tasks.get_task_ids(config.PACKAGE_NAME, 'world')
host = sdk_marathon.get_scheduler_host(config.PACKAGE_NAME)
config.bump_world_cpus()
sdk_tasks.kill_task_with_pattern('helloworld.scheduler.Main', host)
sdk_tasks.check_tasks_updated(config.PACKAGE_NAME, 'world', world_ids)
config.check_running()
def test_kill_hello_node():
config.check_running()
hello_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-0')
sdk_tasks.kill_task_with_pattern('hello', 'hello-0-server.hello-world.mesos')
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'hello', hello_ids)
config.check_running()
def test_config_update_then_kill_task_in_node():
# kill 1 of 2 world tasks
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world')
config.bump_world_cpus()
sdk_tasks.kill_task_with_pattern('world', 'world-0-server.{}.mesos'.format(config.SERVICE_NAME))
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world', world_ids)
config.check_running()
def test_config_update_then_scheduler_died():
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world')
host = sdk_marathon.get_scheduler_host(config.SERVICE_NAME)
config.bump_world_cpus()
sdk_tasks.kill_task_with_pattern('helloworld.scheduler.Main', host)
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world', world_ids)
config.check_running()
def test_config_update_then_kill_task_in_node():
# kill 1 of 2 world tasks
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world')
config.bump_world_cpus()
sdk_tasks.kill_task_with_pattern('world', 'world-0-server.{}.mesos'.format(config.SERVICE_NAME))
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world', world_ids)
config.check_running()
def test_kill_hello_node():
config.check_running()
hello_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-0')
sdk_tasks.kill_task_with_pattern('hello', 'hello-0-server.hello-world.mesos')
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'hello-0', hello_ids)
config.check_running()
def test_config_update_then_executor_killed():
world_ids = sdk_tasks.get_task_ids(config.PACKAGE_NAME, 'world')
config.bump_world_cpus()
sdk_tasks.kill_task_with_pattern(
'helloworld.executor.Main',
'world-0-server.{}.mesos'.format(config.PACKAGE_NAME))
sdk_tasks.check_tasks_updated(config.PACKAGE_NAME, 'world', world_ids)
config.check_running()
def test_config_update_then_kill_task_in_node():
# kill 1 of 2 world tasks
world_ids = tasks.get_task_ids(PACKAGE_NAME, 'world')
bump_world_cpus()
tasks.kill_task_with_pattern(
'world', 'world-0-server.{}.mesos'.format(PACKAGE_NAME))
tasks.check_tasks_updated(PACKAGE_NAME, 'world', world_ids)
check_running()
def test_kill_journal_node():
journal_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'journal-0')
name_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'name')
data_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'data')
sdk_tasks.kill_task_with_pattern('journalnode', sdk_hosts.system_host(FOLDERED_SERVICE_NAME, 'journal-0-node'))
config.expect_recovery(service_name=FOLDERED_SERVICE_NAME)
sdk_tasks.check_tasks_updated(FOLDERED_SERVICE_NAME, 'journal', journal_ids)
sdk_tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, 'name', name_ids)
sdk_tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, 'data', data_ids)
def test_kill_name_node():
name_ids = tasks.get_task_ids(PACKAGE_NAME, 'name-0')
journal_ids = tasks.get_task_ids(PACKAGE_NAME, 'journal')
data_ids = tasks.get_task_ids(PACKAGE_NAME, 'data')
tasks.kill_task_with_pattern('namenode', 'name-0-node.hdfs.mesos')
check_healthy()
tasks.check_tasks_updated(PACKAGE_NAME, 'name', name_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'journal', journal_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'data', data_ids)
def test_kill_name_node():
name_ids = sdk_tasks.get_task_ids(config.FOLDERED_SERVICE_NAME, 'name-0')
journal_ids = sdk_tasks.get_task_ids(config.FOLDERED_SERVICE_NAME, 'journal')
data_ids = sdk_tasks.get_task_ids(config.FOLDERED_SERVICE_NAME, 'data')
sdk_tasks.kill_task_with_pattern('namenode', sdk_hosts.system_host(config.FOLDERED_SERVICE_NAME, 'name-0-node'))
config.expect_recovery(service_name=config.FOLDERED_SERVICE_NAME)
sdk_tasks.check_tasks_updated(config.FOLDERED_SERVICE_NAME, 'name', name_ids)
sdk_tasks.check_tasks_not_updated(config.FOLDERED_SERVICE_NAME, 'journal', journal_ids)
sdk_tasks.check_tasks_not_updated(config.FOLDERED_SERVICE_NAME, 'data', data_ids)
def test_kill_journal_node():
foldered_name = sdk_utils.get_foldered_name(config.SERVICE_NAME)
journal_ids = sdk_tasks.get_task_ids(foldered_name, 'journal-0')
name_ids = sdk_tasks.get_task_ids(foldered_name, 'name')
data_ids = sdk_tasks.get_task_ids(foldered_name, 'data')
sdk_tasks.kill_task_with_pattern('journalnode', sdk_hosts.system_host(foldered_name, 'journal-0-node'))
config.expect_recovery(service_name=foldered_name)
sdk_tasks.check_tasks_updated(foldered_name, 'journal', journal_ids)
sdk_tasks.check_tasks_not_updated(foldered_name, 'name', name_ids)
sdk_tasks.check_tasks_not_updated(foldered_name, 'data', data_ids)
def test_kill_all_journalnodes():
journal_ids = tasks.get_task_ids(PACKAGE_NAME, 'journal')
data_ids = tasks.get_task_ids(PACKAGE_NAME, 'data')
for host in shakedown.get_service_ips(PACKAGE_NAME):
tasks.kill_task_with_pattern('journalnode', host)
check_healthy()
# name nodes fail and restart, so don't check those
tasks.check_tasks_updated(PACKAGE_NAME, 'journal', journal_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'data', data_ids)
def test_kill_all_datanodes():
journal_ids = tasks.get_task_ids(PACKAGE_NAME, 'journal')
name_ids = tasks.get_task_ids(PACKAGE_NAME, 'name')
data_ids = tasks.get_task_ids(PACKAGE_NAME, 'data')
for host in shakedown.get_service_ips(PACKAGE_NAME):
tasks.kill_task_with_pattern('datanode', host)
check_healthy()
tasks.check_tasks_updated(PACKAGE_NAME, 'data', data_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'journal', journal_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'name', name_ids)
def test_kill_data_node():
data_ids = tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'data-0')
journal_ids = tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'journal')
name_ids = tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'name')
tasks.kill_task_with_pattern(
'datanode', hosts.system_host(FOLDERED_SERVICE_NAME, 'data-0-node'))
check_healthy()
tasks.check_tasks_updated(FOLDERED_SERVICE_NAME, 'data', data_ids)
tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, 'journal',
journal_ids)
tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, 'name', name_ids)
def test_kill_all_journalnodes():
journal_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'journal')
data_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'data')
for host in shakedown.get_service_ips(FOLDERED_SERVICE_NAME):
sdk_tasks.kill_task_with_pattern('journalnode', host)
expect_recovery()
# name nodes fail and restart, so don't check those
sdk_tasks.check_tasks_updated(FOLDERED_SERVICE_NAME, 'journal',
journal_ids)
sdk_tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, 'data', data_ids)
def test_kill_all_namenodes():
journal_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'journal')
name_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'name')
data_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'data')
for host in shakedown.get_service_ips(FOLDERED_SERVICE_NAME):
sdk_tasks.kill_task_with_pattern('namenode', host)
expect_recovery()
sdk_tasks.check_tasks_updated(FOLDERED_SERVICE_NAME, 'name', name_ids)
sdk_tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, 'journal',
journal_ids)
sdk_tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, 'data', data_ids)
def test_kill_journal_node():
check_healthy()
journal_ids = tasks.get_task_ids(PACKAGE_NAME, 'journal-0')
name_ids = tasks.get_task_ids(PACKAGE_NAME, 'name')
zkfc_ids = tasks.get_task_ids(PACKAGE_NAME, 'zkfc')
data_ids = tasks.get_task_ids(PACKAGE_NAME, 'data')
tasks.kill_task_with_pattern('journalnode', 'journal-0-node.hdfs.mesos')
tasks.check_tasks_updated(PACKAGE_NAME, 'journal', journal_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'name', name_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'zkfc', zkfc_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'data', data_ids)
check_healthy()
def test_integrity_on_name_node_failure():
"""
The first name node (name-0-node) is the active name node by default when HDFS gets installed.
This test checks that it is possible to write and read data after the first name node fails.
"""
tasks.kill_task_with_pattern("NameNode", 'name-0-node.hdfs.mesos')
time.sleep(1) # give NameNode a chance to die
shakedown.wait_for(lambda: write_data_to_hdfs("data-0-node.hdfs.mesos", TEST_FILE_2_NAME), HDFS_CMD_TIMEOUT_SEC)
shakedown.wait_for(lambda: read_data_from_hdfs("data-2-node.hdfs.mesos", TEST_FILE_2_NAME), HDFS_CMD_TIMEOUT_SEC)
check_healthy()
def test_integrity_on_data_node_failure():
shakedown.wait_for(lambda: write_data_to_hdfs("data-0-node.hdfs.mesos", TEST_FILE_1_NAME), HDFS_CMD_TIMEOUT_SEC)
# gives chance for write to succeed and replication to occur
time.sleep(5)
tasks.kill_task_with_pattern("DataNode", 'data-0-node.hdfs.mesos')
tasks.kill_task_with_pattern("DataNode", 'data-1-node.hdfs.mesos')
time.sleep(1) # give DataNode a chance to die
shakedown.wait_for(lambda: read_data_from_hdfs("data-2-node.hdfs.mesos", TEST_FILE_1_NAME), HDFS_CMD_TIMEOUT_SEC)
check_healthy()
def test_integrity_on_name_node_failure():
"""
The first name node (name-0-node) is the active name node by default when HDFS gets installed.
This test checks that it is possible to write and read data after the first name node fails.
"""
tasks.kill_task_with_pattern("NameNode", 'name-0-node.hdfs.mesos')
time.sleep(1) # give NameNode a chance to die
shakedown.wait_for(lambda: write_data_to_hdfs("data-0-node.hdfs.mesos", TEST_FILE_2_NAME), HDFS_CMD_TIMEOUT_SEC)
shakedown.wait_for(lambda: read_data_from_hdfs("data-2-node.hdfs.mesos", TEST_FILE_2_NAME), HDFS_CMD_TIMEOUT_SEC)
check_running()
def test_integrity_on_data_node_failure():
shakedown.wait_for(lambda: write_data_to_hdfs("data-0-node.hdfs.mesos", TEST_FILE_1_NAME), HDFS_CMD_TIMEOUT_SEC)
# gives chance for write to succeed and replication to occur
time.sleep(5)
tasks.kill_task_with_pattern("DataNode", 'data-0-node.hdfs.mesos')
tasks.kill_task_with_pattern("DataNode", 'data-1-node.hdfs.mesos')
time.sleep(1) # give DataNode a chance to die
shakedown.wait_for(lambda: read_data_from_hdfs("data-2-node.hdfs.mesos", TEST_FILE_1_NAME), HDFS_CMD_TIMEOUT_SEC)
check_running()
def test_kill_data_node():
data_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'data-0')
journal_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'journal')
name_ids = sdk_tasks.get_task_ids(FOLDERED_SERVICE_NAME, 'name')
sdk_tasks.kill_task_with_pattern(
'datanode', sdk_hosts.system_host(FOLDERED_SERVICE_NAME,
'data-0-node'))
expect_recovery()
sdk_tasks.check_tasks_updated(FOLDERED_SERVICE_NAME, 'data', data_ids)
sdk_tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, 'journal',
journal_ids)
sdk_tasks.check_tasks_not_updated(FOLDERED_SERVICE_NAME, 'name', name_ids)
def test_integrity_on_data_node_failure():
"""
Verifies proper data replication among data nodes.
"""
# An HDFS write will only successfully return when the data replication has taken place
config.write_data_to_hdfs(config.SERVICE_NAME, config.TEST_FILE_1_NAME)
sdk_tasks.kill_task_with_pattern("DataNode", sdk_hosts.system_host(config.SERVICE_NAME, 'data-0-node'))
sdk_tasks.kill_task_with_pattern("DataNode", sdk_hosts.system_host(config.SERVICE_NAME, 'data-1-node'))
config.read_data_from_hdfs(config.SERVICE_NAME, config.TEST_FILE_1_NAME)
config.check_healthy(service_name=config.SERVICE_NAME)
def test_integrity_on_name_node_failure():
"""
The first name node (name-0-node) is the active name node by default when HDFS gets installed.
This test checks that it is possible to write and read data after the first name node fails.
"""
tasks.kill_task_with_pattern(
"NameNode", hosts.system_host(FOLDERED_SERVICE_NAME, 'name-0-node'))
time.sleep(1) # give NameNode a chance to die
write_some_data('data-0-node', TEST_FILE_2_NAME)
read_some_data('data-2-node', TEST_FILE_2_NAME)
check_healthy()
def test_kill_all_journalnodes():
check_running()
journal_ids = tasks.get_task_ids(PACKAGE_NAME, 'journal')
name_ids = tasks.get_task_ids(PACKAGE_NAME, 'name')
zkfc_ids = tasks.get_task_ids(PACKAGE_NAME, 'zkfc')
data_ids = tasks.get_task_ids(PACKAGE_NAME, 'data')
for host in shakedown.get_service_ips(PACKAGE_NAME):
tasks.kill_task_with_pattern('journalnode', host)
tasks.check_tasks_updated(PACKAGE_NAME, 'journal', journal_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'name', name_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'zkfc', zkfc_ids)
tasks.check_tasks_not_updated(PACKAGE_NAME, 'data', data_ids)
check_running()
def test_integrity_on_data_node_failure():
write_some_data('data-0-node', TEST_FILE_1_NAME)
# gives chance for write to succeed and replication to occur
time.sleep(9)
tasks.kill_task_with_pattern(
"DataNode", hosts.system_host(FOLDERED_SERVICE_NAME, 'data-0-node'))
tasks.kill_task_with_pattern(
"DataNode", hosts.system_host(FOLDERED_SERVICE_NAME, 'data-1-node'))
time.sleep(1) # give DataNode a chance to die
read_some_data('data-2-node', TEST_FILE_1_NAME)
check_healthy()
def test_config_updates_then_all_executors_killed():
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world')
hosts = shakedown.get_service_ips(config.SERVICE_NAME)
config.bump_world_cpus()
[sdk_tasks.kill_task_with_pattern('helloworld.executor.Main', h) for h in hosts]
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world', world_ids)
config.check_running()
def test_kill_essential():
'''kill the essential task, verify that both tasks are relaunched against a matching executor'''
verify_shared_executor('hello-0')
old_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-0')
assert len(old_ids) == 2
sdk_tasks.kill_task_with_pattern(
'shared-volume/essential', # hardcoded in cmd, see yml
sdk_hosts.system_host(config.SERVICE_NAME, 'hello-0-essential'))
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'hello-0', old_ids) # wait for ids to change...
sdk_plan.wait_for_completed_recovery(config.SERVICE_NAME) # ...and for tasks to be up and running
# the first verify_shared_executor call deleted the files. both should have come back via the relaunch.
verify_shared_executor('hello-0', delete_files=False) # leave files as-is for the next test
def test_config_updates_then_all_executors_killed():
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world')
hosts = shakedown.get_service_ips(config.SERVICE_NAME)
config.bump_world_cpus()
[sdk_tasks.kill_task_with_pattern('helloworld.executor.Main', h) for h in hosts]
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world', world_ids)
config.check_running()
def test_config_update_then_kill_all_task_in_node():
# kill both world tasks
world_ids = sdk_tasks.get_task_ids(config.PACKAGE_NAME, 'world')
hosts = shakedown.get_service_ips(config.PACKAGE_NAME)
config.bump_world_cpus()
[sdk_tasks.kill_task_with_pattern('world', h) for h in hosts]
sdk_tasks.check_tasks_updated(config.PACKAGE_NAME, 'world', world_ids)
config.check_running()
def test_config_update_then_kill_all_task_in_node():
# kill both world tasks
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world')
hosts = shakedown.get_service_ips(config.SERVICE_NAME)
config.bump_world_cpus()
[sdk_tasks.kill_task_with_pattern('world', h) for h in hosts]
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world', world_ids)
config.check_running()
def test_kill_nonessential():
'''kill the nonessential task, verify that the nonessential task is relaunched against the same executor as before'''
verify_shared_executor('hello-0')
old_essential_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-0-essential')
assert len(old_essential_ids) == 1
old_nonessential_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello-0-nonessential')
assert len(old_nonessential_ids) == 1
sdk_tasks.kill_task_with_pattern(
'shared-volume/nonessential', # hardcoded in cmd, see yml
sdk_hosts.system_host(config.SERVICE_NAME, 'hello-0-nonessential'))
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'hello-0-nonessential', old_nonessential_ids)
sdk_plan.wait_for_completed_recovery(config.SERVICE_NAME)
sdk_tasks.check_tasks_not_updated(config.SERVICE_NAME, 'hello-0-essential', old_essential_ids)
# the first verify_shared_executor call deleted the files. only the nonessential file came back via its relaunch.
verify_shared_executor('hello-0', expected_files=['nonessential'])
def test_integrity_on_name_node_failure():
"""
The first name node (name-0-node) is the active name node by default when HDFS gets installed.
This test checks that it is possible to write and read data after the active name node fails
so as to verify a failover sustains expected functionality.
"""
active_name_node = config.get_active_name_node(config.SERVICE_NAME)
sdk_tasks.kill_task_with_pattern("NameNode", sdk_hosts.system_host(config.SERVICE_NAME, active_name_node))
predicted_active_name_node = "name-1-node"
if active_name_node == "name-1-node":
predicted_active_name_node = "name-0-node"
wait_for_failover_to_complete(predicted_active_name_node)
config.write_data_to_hdfs(config.SERVICE_NAME, config.TEST_FILE_2_NAME)
config.read_data_from_hdfs(config.SERVICE_NAME, config.TEST_FILE_2_NAME)
config.check_healthy(service_name=config.SERVICE_NAME)
def test_kill_scheduler():
sdk_tasks.kill_task_with_pattern('hdfs.scheduler.Main', shakedown.get_service_ips('marathon').pop())
config.check_healthy(service_name=sdk_utils.get_foldered_name(config.SERVICE_NAME))
def test_kill_scheduler():
sdk_tasks.kill_task_with_pattern('hdfs.scheduler.Main', shakedown.get_service_ips('marathon').pop())
config.check_healthy(service_name=FOLDERED_SERVICE_NAME)
def test_all_executors_killed():
for host in shakedown.get_service_ips(config.SERVICE_NAME):
sdk_tasks.kill_task_with_pattern('helloworld.executor.Main', host)
config.check_running()
def test_scheduler_died():
sdk_tasks.kill_task_with_pattern('helloworld.scheduler.Main', sdk_marathon.get_scheduler_host(config.SERVICE_NAME))
config.check_running()
def test_master_killed():
sdk_tasks.kill_task_with_pattern('mesos-master')
config.check_running()
def test_kill_scheduler():
sdk_tasks.kill_task_with_pattern(
'hdfs.scheduler.Main',
shakedown.get_service_ips('marathon').pop())
check_healthy()
def test_config_update_then_master_killed():
world_ids = sdk_tasks.get_task_ids(config.PACKAGE_NAME, 'world')
config.bump_world_cpus()
sdk_tasks.kill_task_with_pattern('mesos-master')
sdk_tasks.check_tasks_updated(config.PACKAGE_NAME, 'world', world_ids)
config.check_running()
def test_config_update_then_zk_killed():
hello_ids = sdk_tasks.get_task_ids(config.PACKAGE_NAME, 'hello')
config.bump_hello_cpus()
sdk_tasks.kill_task_with_pattern('zookeeper')
sdk_tasks.check_tasks_updated(config.PACKAGE_NAME, 'hello', hello_ids)
config.check_running()
def test_zk_killed():
sdk_tasks.kill_task_with_pattern('zookeeper')
config.check_running()
def test_config_update_then_executor_killed():
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world')
config.bump_world_cpus()
sdk_tasks.kill_task_with_pattern('helloworld.executor.Main', 'world-0-server.{}.mesos'.format(config.SERVICE_NAME))
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world', world_ids)
config.check_running()
def test_config_update_then_master_killed():
world_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'world')
config.bump_world_cpus()
sdk_tasks.kill_task_with_pattern('mesos-master')
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'world', world_ids)
config.check_running()
def test_config_update_then_zk_killed():
hello_ids = sdk_tasks.get_task_ids(config.SERVICE_NAME, 'hello')
config.bump_hello_cpus()
sdk_tasks.kill_task_with_pattern('zookeeper')
sdk_tasks.check_tasks_updated(config.SERVICE_NAME, 'hello', hello_ids)
config.check_running()
def test_kill_hello_node():
hello_ids = tasks.get_task_ids(PACKAGE_NAME, 'hello-0')
tasks.kill_task_with_pattern('hello', 'hello-0-server.hello-world.mesos')
tasks.check_tasks_updated(PACKAGE_NAME, 'hello', hello_ids)
check_running()