def test_pod_with_container_bridge_network():
"""Tests creation of a pod with a "container/bridge" network, and its HTTP endpoint accessibility."""
pod_def = pods.container_bridge_pod()
pod_id = pod_def['id']
# In strict mode all tasks are started as user `nobody` by default and `nobody`
# doesn't have permissions to write to /var/log within the container.
if shakedown.ee_version() == 'strict':
pod_def['user'] = '******'
common.add_dcos_marathon_user_acls()
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
task = common.task_by_name(common.get_pod_tasks(pod_id), "nginx")
network_info = common.running_status_network_info(task['statuses'])
assert network_info['name'] == "mesos-bridge", \
"The network is {}, but mesos-bridge was expected".format(network_info['name'])
# get the port on the host
port = task['discovery']['ports']['ports'][0]['number']
# the agent IP:port will be routed to the bridge IP:port
# test against the agent_ip, however it is hard to get.. translating from
# slave_id
agent_ip = common.agent_hostname_by_id(task['slave_id'])
assert agent_ip is not None, "Failed to get the agent IP address"
container_ip = network_info['ip_addresses'][0]['ip_address']
assert agent_ip != container_ip, "The container IP address is the same as the agent one"
url = "http://{}:{}/".format(agent_ip, port)
common.assert_http_code(url)
def test_pod_with_persistent_volume():
pod_def = pods.persistent_volume_pod()
pod_id = pod_def['id']
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
host = common.running_status_network_info(
tasks[0]['statuses'])['ip_addresses'][0]['ip_address']
port1 = tasks[0]['discovery']['ports']['ports'][0]["number"]
port2 = tasks[1]['discovery']['ports']['ports'][0]["number"]
path1 = tasks[0]['container']['volumes'][0]['container_path']
path2 = tasks[1]['container']['volumes'][0]['container_path']
print(host, port1, port2, path1, path2)
@retrying.retry(wait_fixed=1000,
stop_max_attempt_number=60,
retry_on_exception=common.ignore_exception)
def check_http_endpoint(port, path):
cmd = "curl {}:{}/{}/foo".format(host, port, path)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert data == 'hello\n', "'{}' was not equal to hello\\n".format(data)
check_http_endpoint(port1, path1)
check_http_endpoint(port2, path2)
def test_pod_with_persistent_volume():
pod_def = pods.persistent_volume_pod()
pod_id = pod_def['id']
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
host = common.running_status_network_info(
tasks[0]['statuses'])['ip_addresses'][0]['ip_address']
port1 = tasks[0]['discovery']['ports']['ports'][0]["number"]
port2 = tasks[1]['discovery']['ports']['ports'][0]["number"]
dir1 = tasks[0]['container']['volumes'][0]['container_path']
dir2 = tasks[1]['container']['volumes'][0]['container_path']
print(host, port1, port2, dir1, dir2)
time.sleep(1)
cmd = "curl {}:{}/{}/foo".format(host, port1, dir1)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert data == 'hello\n', "'{}' was not equal to hello\\n".format(data)
cmd = "curl {}:{}/{}/foo".format(host, port2, dir2)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert data == 'hello\n', "'{}' was not equal to hello\\n".format(data)
def test_pod_with_container_network():
"""Tests creation of a pod with a "container" network, and its HTTP endpoint accessibility."""
pod_def = pods.container_net_pod()
pod_id = pod_def['id']
# In strict mode all tasks are started as user `nobody` by default and `nobody`
# doesn't have permissions to write to /var/log within the container.
if shakedown.ee_version() == 'strict':
pod_def['user'] = '******'
common.add_dcos_marathon_user_acls()
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
task = common.task_by_name(common.get_pod_tasks(pod_id), "nginx")
network_info = common.running_status_network_info(task['statuses'])
assert network_info['name'] == "dcos", \
"The network name is {}, but 'dcos' was expected".format(network_info['name'])
container_ip = network_info['ip_addresses'][0]['ip_address']
assert container_ip is not None, "No IP address has been assigned to the pod's container"
url = "http://{}:80/".format(container_ip)
common.assert_http_code(url)
def test_pod_with_persistent_volume():
pod_def = pods.persistent_volume_pod()
pod_id = pod_def['id']
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
host = common.running_status_network_info(tasks[0]['statuses'])['ip_addresses'][0]['ip_address']
port1 = tasks[0]['discovery']['ports']['ports'][0]["number"]
port2 = tasks[1]['discovery']['ports']['ports'][0]["number"]
path1 = tasks[0]['container']['volumes'][0]['container_path']
path2 = tasks[1]['container']['volumes'][0]['container_path']
print(host, port1, port2, path1, path2)
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=60, retry_on_exception=common.ignore_exception)
def check_http_endpoint(port, path):
cmd = "curl {}:{}/{}/foo".format(host, port, path)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert data == 'hello\n', "'{}' was not equal to hello\\n".format(data)
check_http_endpoint(port1, path1)
check_http_endpoint(port2, path2)
def test_pod_with_container_bridge_network():
"""Tests creation of a pod with a "container/bridge" network, and its HTTP endpoint accessibility."""
pod_def = pods.container_bridge_pod()
pod_id = pod_def['id']
# In strict mode all tasks are started as user `nobody` by default and `nobody`
# doesn't have permissions to write to /var/log within the container.
if shakedown.ee_version() == 'strict':
pod_def['user'] = '******'
common.add_dcos_marathon_user_acls()
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
task = common.task_by_name(common.get_pod_tasks(pod_id), "nginx")
network_info = common.running_status_network_info(task['statuses'])
assert network_info['name'] == "mesos-bridge", \
"The network is {}, but mesos-bridge was expected".format(network_info['name'])
# get the port on the host
port = task['discovery']['ports']['ports'][0]['number']
# the agent IP:port will be routed to the bridge IP:port
# test against the agent_ip, however it is hard to get.. translating from
# slave_id
agent_ip = common.agent_hostname_by_id(task['slave_id'])
assert agent_ip is not None, "Failed to get the agent IP address"
container_ip = network_info['ip_addresses'][0]['ip_address']
assert agent_ip != container_ip, "The container IP address is the same as the agent one"
url = "http://{}:{}/".format(agent_ip, port)
common.assert_http_code(url)
def test_pod_with_container_network():
"""Tests creation of a pod with a "container" network, and its HTTP endpoint accessibility."""
pod_def = pods.container_net_pod()
pod_id = pod_def['id']
# In strict mode all tasks are started as user `nobody` by default and `nobody`
# doesn't have permissions to write to /var/log within the container.
if shakedown.ee_version() == 'strict':
pod_def['user'] = '******'
common.add_dcos_marathon_user_acls()
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
task = common.task_by_name(common.get_pod_tasks(pod_id), "nginx")
network_info = common.running_status_network_info(task['statuses'])
assert network_info['name'] == "dcos", \
"The network name is {}, but 'dcos' was expected".format(network_info['name'])
container_ip = network_info['ip_addresses'][0]['ip_address']
assert container_ip is not None, "No IP address has been assigned to the pod's container"
url = "http://{}:80/".format(container_ip)
common.assert_http_code(url)
def test_pod_with_persistent_volume():
pod_def = pods.persistent_volume_pod()
pod_id = pod_def['id']
client = marathon.create_client()
client.add_pod(pod_def)
deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
host = common.running_status_network_info(tasks[0]['statuses'])['ip_addresses'][0]['ip_address']
# Container with the name 'container1' appends its taskId to the file. So we search for the
# taskId of that container which is not always the tasks[0]
expected_data = next((t['id'] for t in tasks if t['name'] == 'container1'), None)
assert expected_data, f"Hasn't found a container with the name 'container1' in the pod {tasks}"
port1 = tasks[0]['discovery']['ports']['ports'][0]["number"]
port2 = tasks[1]['discovery']['ports']['ports'][0]["number"]
path1 = tasks[0]['container']['volumes'][0]['container_path']
path2 = tasks[1]['container']['volumes'][0]['container_path']
logger.info('Deployd two containers on {}:{}/{} and {}:{}/{}'.format(host, port1, path1, host, port2, path2))
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=60, retry_on_exception=common.ignore_exception)
def check_http_endpoint(port, path, expected):
cmd = "curl {}:{}/{}/foo".format(host, port, path)
run, data = run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert expected in data, "'{}' was not found in '{}'".format(data, expected)
check_http_endpoint(port1, path1, expected_data)
check_http_endpoint(port2, path2, expected_data)
def wait_for_status_network_info():
tasks = common.get_pod_tasks(pod_id)
# the following command throws exceptions if there are no tasks in TASK_RUNNING state
common.running_status_network_info(tasks[0]['statuses'])
def test_pod_with_persistent_volume_recovers():
pod_def = pods.persistent_volume_pod()
pod_id = pod_def['id']
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
assert len(
tasks
) == 2, "The number of pod tasks is {}, but is expected to be 2".format(
len(tasks))
@retrying.retry(wait_fixed=1000,
stop_max_attempt_number=30,
retry_on_exception=common.ignore_exception)
def wait_for_status_network_info():
tasks = common.get_pod_tasks(pod_id)
# the following command throws exceptions if there are no tasks in TASK_RUNNING state
common.running_status_network_info(tasks[0]['statuses'])
wait_for_status_network_info()
host = common.running_status_network_info(
tasks[0]['statuses'])['ip_addresses'][0]['ip_address']
task_id1 = tasks[0]['id']
task_id2 = tasks[1]['id']
@retrying.retry(wait_fixed=1000,
stop_max_attempt_number=30,
retry_on_exception=common.ignore_exception)
def kill_task(host, pattern):
pids = common.kill_process_on_host(host, pattern)
assert len(
pids) != 0, "no task got killed on {} for pattern {}".format(
host, pattern)
kill_task(host, '[h]ttp\\.server')
@retrying.retry(wait_fixed=1000,
stop_max_attempt_number=30,
retry_on_exception=common.ignore_exception)
def wait_for_pod_recovery():
tasks = common.get_pod_tasks(pod_id)
assert len(
tasks
) == 2, "The number of tasks is {} after recovery, but 2 was expected".format(
len(tasks))
old_task_ids = [task_id1, task_id2]
new_task_id1 = tasks[0]['id']
new_task_id2 = tasks[1]['id']
assert new_task_id1 not in old_task_ids, \
"The task ID has not changed, and is still {}".format(new_task_id1)
assert new_task_id2 not in old_task_ids, \
"The task ID has not changed, and is still {}".format(new_task_id2)
wait_for_pod_recovery()
wait_for_status_network_info()
tasks = common.get_pod_tasks(pod_id)
assert host == common.running_status_network_info(tasks[0]['statuses'])['ip_addresses'][0]['ip_address'], \
"the pod has been restarted on another host"
port1 = tasks[0]['discovery']['ports']['ports'][0]["number"]
port2 = tasks[1]['discovery']['ports']['ports'][0]["number"]
path1 = tasks[0]['container']['volumes'][0]['container_path']
path2 = tasks[1]['container']['volumes'][0]['container_path']
print(host, port1, port2, path1, path2)
@retrying.retry(wait_fixed=1000,
stop_max_attempt_number=30,
retry_on_exception=common.ignore_exception)
def check_data(port, path):
cmd = "curl {}:{}/{}/foo".format(host, port, path)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert 'hello\nhello\n' in data, "'hello\nhello\n' not found in '{}'n".format(
data)
check_data(port1, path1)
check_data(port2, path2)
def wait_for_status_network_info():
tasks = common.get_pod_tasks(pod_id)
# the following command throws exceptions if there are no tasks in TASK_RUNNING state
common.running_status_network_info(tasks[0]['statuses'])
def test_pod_with_persistent_volume_recovers():
pod_def = pods.persistent_volume_pod()
pod_id = pod_def['id']
client = marathon.create_client()
client.add_pod(pod_def)
common.deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
assert len(tasks) == 2, "The number of pod tasks is {}, but is expected to be 2".format(len(tasks))
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def wait_for_status_network_info():
tasks = common.get_pod_tasks(pod_id)
# the following command throws exceptions if there are no tasks in TASK_RUNNING state
common.running_status_network_info(tasks[0]['statuses'])
wait_for_status_network_info()
host = common.running_status_network_info(tasks[0]['statuses'])['ip_addresses'][0]['ip_address']
task_id1 = tasks[0]['id']
task_id2 = tasks[1]['id']
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def kill_task(host, pattern):
pids = common.kill_process_on_host(host, pattern)
assert len(pids) != 0, "no task got killed on {} for pattern {}".format(host, pattern)
kill_task(host, '[h]ttp\\.server')
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def wait_for_pod_recovery():
tasks = common.get_pod_tasks(pod_id)
assert len(tasks) == 2, "The number of tasks is {} after recovery, but 2 was expected".format(len(tasks))
old_task_ids = [task_id1, task_id2]
new_task_id1 = tasks[0]['id']
new_task_id2 = tasks[1]['id']
assert new_task_id1 not in old_task_ids, \
"The task ID has not changed, and is still {}".format(new_task_id1)
assert new_task_id2 not in old_task_ids, \
"The task ID has not changed, and is still {}".format(new_task_id2)
wait_for_pod_recovery()
wait_for_status_network_info()
tasks = common.get_pod_tasks(pod_id)
assert host == common.running_status_network_info(tasks[0]['statuses'])['ip_addresses'][0]['ip_address'], \
"the pod has been restarted on another host"
port1 = tasks[0]['discovery']['ports']['ports'][0]["number"]
port2 = tasks[1]['discovery']['ports']['ports'][0]["number"]
path1 = tasks[0]['container']['volumes'][0]['container_path']
path2 = tasks[1]['container']['volumes'][0]['container_path']
print(host, port1, port2, path1, path2)
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_data(port, path):
cmd = "curl {}:{}/{}/foo".format(host, port, path)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert 'hello\nhello\n' in data, "'hello\nhello\n' not found in '{}'n".format(data)
check_data(port1, path1)
check_data(port2, path2)
def test_pod_with_persistent_volume_recovers():
pod_def = pods.persistent_volume_pod()
pod_id = pod_def['id']
client = marathon.create_client()
client.add_pod(pod_def)
deployment_wait(service_id=pod_id)
tasks = common.get_pod_tasks(pod_id)
assert len(tasks) == 2, "The number of pod tasks is {}, but is expected to be 2".format(len(tasks))
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def wait_for_status_network_info():
tasks = common.get_pod_tasks(pod_id)
# the following command throws exceptions if there are no tasks in TASK_RUNNING state
common.running_status_network_info(tasks[0]['statuses'])
wait_for_status_network_info()
host = common.running_status_network_info(tasks[0]['statuses'])['ip_addresses'][0]['ip_address']
task_id1 = tasks[0]['id']
task_id2 = tasks[1]['id']
port1 = tasks[0]['discovery']['ports']['ports'][0]["number"]
path1 = tasks[0]['container']['volumes'][0]['container_path']
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_data(port, path, expected):
cmd = "curl {}:{}/{}/foo".format(host, port, path)
run, data = run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert expected in data, "{} not found in '{}'n".format(expected, data)
# Container with the name 'container1' appends its taskId to the file. So we search for the
# taskId of that container which is not always the tasks[0]
expected_data1 = next((t['id'] for t in tasks if t['name'] == 'container1'), None)
assert expected_data1, f"Hasn't found a container with the name 'container1' in the pod {tasks}"
check_data(port1, path1, expected_data1)
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def kill_task(host, pattern):
pids = common.kill_process_on_host(host, pattern)
assert len(pids) != 0, "no task got killed on {} for pattern {}".format(host, pattern)
kill_task(host, '[h]ttp\\.server')
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def wait_for_pod_recovery():
tasks = common.get_pod_tasks(pod_id)
assert len(tasks) == 2, "The number of tasks is {} after recovery, but 2 was expected".format(len(tasks))
old_task_ids = [task_id1, task_id2]
new_task_id1 = tasks[0]['id']
new_task_id2 = tasks[1]['id']
assert new_task_id1 not in old_task_ids, \
"The task ID has not changed, and is still {}".format(new_task_id1)
assert new_task_id2 not in old_task_ids, \
"The task ID has not changed, and is still {}".format(new_task_id2)
wait_for_pod_recovery()
wait_for_status_network_info()
tasks = common.get_pod_tasks(pod_id)
assert host == common.running_status_network_info(tasks[0]['statuses'])['ip_addresses'][0]['ip_address'], \
"the pod has been restarted on another host"
port1 = tasks[0]['discovery']['ports']['ports'][0]["number"]
port2 = tasks[1]['discovery']['ports']['ports'][0]["number"]
path1 = tasks[0]['container']['volumes'][0]['container_path']
path2 = tasks[1]['container']['volumes'][0]['container_path']
logger.info('Deployd two containers on {}:{}/{} and {}:{}/{}'.format(host, port1, path1, host, port2, path2))
# Container with the name 'container1' appends its taskId to the file. So we search for the
# taskId of that container which is not always the tasks[0]
expected_data2 = next((t['id'] for t in tasks if t['name'] == 'container1'), None)
assert expected_data2, f"Hasn't found a container with the name 'container1' in the pod {tasks}"
check_data(port1, path1, f"{expected_data1}\n{expected_data2}\n")
check_data(port2, path2, f"{expected_data1}\n{expected_data2}\n")