def partition_agent(hostname):
"""Partition a node from all network traffic except for SSH and loopback"""
shakedown.copy_file_to_agent(
hostname, "{}/net-services-agent.sh".format(fixture_dir()))
print("partitioning {}".format(hostname))
shakedown.run_command_on_agent(hostname, 'sh net-services-agent.sh fail')
def test_default_user():
"""Install the Marathon package for DC/OS.
"""
# launch unique-sleep
application_json = get_resource("{}/unique-sleep.json".format(fixture_dir()))
client = marathon.create_client()
client.add_app(application_json)
app = client.get_app(application_json['id'])
assert app['user'] == None
# wait for deployment to finish
tasks = client.get_tasks("unique-sleep")
host = tasks[0]['host']
assert run_command_on_agent(host,"ps aux | grep '[s]leep ' | awk '{if ($1 !=\"root\") exit 1;}'")
def test_mom_with_network_failure_bounce_master():
"""Marathon on Marathon (MoM) tests for DC/OS with network failures simulated by
knocking out ports
"""
# get MoM ip
mom_ip = ip_of_mom()
print("MoM IP: {}".format(mom_ip))
app_def = get_resource("{}/large-sleep.json".format(fixture_dir()))
with shakedown.marathon_on_marathon():
client = marathon.create_client()
client.add_app(app_def)
shakedown.wait_for_task("marathon-user", "sleep")
tasks = client.get_tasks('sleep')
original_sleep_task_id = tasks[0]["id"]
task_ip = tasks[0]['host']
print("\nTask IP: " + task_ip)
# PR for network partitioning in shakedown makes this better
# take out the net
partition_agent(mom_ip)
partition_agent(task_ip)
# wait for a min
time.sleep(timedelta(minutes=1).total_seconds())
# bounce master
shakedown.run_command_on_master("sudo systemctl restart dcos-mesos-master")
# bring the net up
reconnect_agent(mom_ip)
reconnect_agent(task_ip)
time.sleep(timedelta(minutes=1).total_seconds())
shakedown.wait_for_service_endpoint('marathon-user',
timedelta(minutes=10).total_seconds())
with shakedown.marathon_on_marathon():
client = marathon.create_client()
shakedown.wait_for_task("marathon-user", "sleep",
timedelta(minutes=10).total_seconds())
tasks = client.get_tasks('sleep')
current_sleep_task_id = tasks[0]["id"]
assert current_sleep_task_id == original_sleep_task_id, "Task ID shouldn't change"
def test_mom_with_network_failure_bounce_master():
"""Marathon on Marathon (MoM) tests for DC/OS with network failures simulated by
knocking out ports
"""
# get MoM ip
mom_ip = ip_of_mom()
print("MoM IP: {}".format(mom_ip))
app_def = get_resource("{}/large-sleep.json".format(fixture_dir()))
with shakedown.marathon_on_marathon():
client = marathon.create_client()
client.add_app(app_def)
shakedown.wait_for_task("marathon-user", "sleep")
tasks = client.get_tasks('sleep')
original_sleep_task_id = tasks[0]["id"]
task_ip = tasks[0]['host']
print("\nTask IP: " + task_ip)
# PR for network partitioning in shakedown makes this better
# take out the net
partition_agent(mom_ip)
partition_agent(task_ip)
# wait for a min
time.sleep(timedelta(minutes=1).total_seconds())
# bounce master
shakedown.run_command_on_master("sudo systemctl restart dcos-mesos-master")
# bring the net up
reconnect_agent(mom_ip)
reconnect_agent(task_ip)
time.sleep(timedelta(minutes=1).total_seconds())
shakedown.wait_for_service_endpoint('marathon-user')
shakedown.wait_for_task("marathon-user", "sleep")
with shakedown.marathon_on_marathon():
client = marathon.create_client()
shakedown.wait_for_task("marathon-user", "sleep")
tasks = client.get_tasks('sleep')
current_sleep_task_id = tasks[0]["id"]
assert current_sleep_task_id == original_sleep_task_id, "Task ID shouldn't change"
def test_network_pinger(test_type, get_pinger_app, dns_format):
""" This test runs a pinger app and a relay app. It retrieves the python app from the
master via the new http service (which will be moving into shakedown). Then a curl call
to the relay will invoke a call to the 2nd pinger app and return back pong to the relay
then back to curl.
It tests that 1 task can network communicate to another task on the given network
It tests inbound and outbound connectivity
test_type param is not used. It is passed so that it is clear which parametrized test
is running or may be failing.
"""
marathon_service_name = get_marathon_service_name()
client = marathon.create_client()
pinger_app = get_pinger_app(
'pinger'
) #add_container_network(common.pinger_localhost_app(), 'dcos')
relay_app = get_pinger_app(
'relay'
) #add_container_network(common.pinger_localhost_app('relay'), 'dcos')
pinger_dns = dns_format.format('pinger', marathon_service_name)
relay_dns = dns_format.format('relay', marathon_service_name)
# test pinger app to master
shakedown.copy_file_to_master(fixture_dir() + "/pinger.py")
with shakedown.master_http_service():
# need to add app with http service in place or it will fail to fetch
client.add_app(pinger_app)
client.add_app(relay_app)
shakedown.deployment_wait()
shakedown.wait_for_dns(relay_dns)
relay_url = 'http://{}:7777/relay-ping?url={}:7777'.format(
relay_dns, pinger_dns)
@retrying.retry
def http_output_check(stop_max_attempt_number=30):
status, output = shakedown.run_command_on_master(
'curl {}'.format(relay_url))
assert status
assert 'Pong /pinger' in output
assert 'Relay from /relay' in output
http_output_check()
def test_default_user():
"""Ensures the default user of a task that is created is started as root.
"""
# launch unique-sleep
application_json = get_resource("{}/unique-sleep.json".format(
fixture_dir()))
client = marathon.create_client()
client.add_app(application_json)
app = client.get_app(application_json['id'])
assert app['user'] is None
# wait for deployment to finish
tasks = client.get_tasks("unique-sleep")
host = tasks[0]['host']
assert run_command_on_agent(
host, "ps aux | grep '[s]leep ' | awk '{if ($1 !=\"root\") exit 1;}'")
def test_default_user():
""" Ensures the default user of a task is started as root. This is the default user.
"""
# launch unique-sleep
application_json = get_resource("{}/unique-sleep.json".format(fixture_dir()))
client = marathon.create_client()
client.add_app(application_json)
app = client.get_app(application_json['id'])
assert app['user'] is None
# wait for deployment to finish
tasks = client.get_tasks("unique-sleep")
host = tasks[0]['host']
assert run_command_on_agent(host, "ps aux | grep '[s]leep ' | awk '{if ($1 !=\"root\") exit 1;}'")
client = marathon.create_client()
client.remove_app("/unique-sleep")
def test_mom_with_network_failure():
"""Marathon on Marathon (MoM) tests for DC/OS with network failures
simulated by knocking out ports
"""
# get MoM ip
mom_ip = ip_of_mom()
print("MoM IP: {}".format(mom_ip))
app_def = get_resource("{}/large-sleep.json".format(fixture_dir()))
with marathon_on_marathon():
client = marathon.create_client()
client.add_app(app_def)
shakedown.wait_for_task("marathon-user", "sleep")
tasks = client.get_tasks('sleep')
original_sleep_task_id = tasks[0]["id"]
task_ip = tasks[0]['host']
# PR for network partitioning in shakedown makes this better
# take out the net
partition_agent(mom_ip)
partition_agent(task_ip)
# wait for a min
service_delay()
# bring the net up
reconnect_agent(mom_ip)
reconnect_agent(task_ip)
service_delay()
shakedown.wait_for_service_endpoint(PACKAGE_APP_ID)
shakedown.wait_for_task("marathon-user", "sleep")
with marathon_on_marathon():
client = marathon.create_client()
shakedown.wait_for_task("marathon-user", "sleep")
tasks = client.get_tasks('sleep')
current_sleep_task_id = tasks[0]["id"]
assert current_sleep_task_id == original_sleep_task_id, "Task ID shouldn't change"
def test_default_user():
""" Ensures the default user of a task is started as root. This is the default user.
"""
# launch unique-sleep
application_json = get_resource("{}/unique-sleep.json".format(fixture_dir()))
client = marathon.create_client()
client.add_app(application_json)
shakedown.deployment_wait()
app = client.get_app(application_json['id'])
user = app.get('user')
assert user is None
# wait for deployment to finish
tasks = client.get_tasks("unique-sleep")
host = tasks[0]['host']
assert shakedown.run_command_on_agent(host, "ps aux | grep '[s]leep ' | awk '{if ($1 !=\"root\") exit 1;}'")
client = marathon.create_client()
client.remove_app("/unique-sleep")
def test_network_pinger(test_type, get_pinger_app, dns_format, marathon_service_name):
""" This test runs a pinger app and a relay app. It retrieves the python app from the
master via the new http service (which will be moving into shakedown). Then a curl call
to the relay will invoke a call to the 2nd pinger app and return back pong to the relay
then back to curl.
It tests that 1 task can network communicate to another task on the given network
It tests inbound and outbound connectivity
test_type param is not used. It is passed so that it is clear which parametrized test
is running or may be failing.
"""
client = marathon.create_client()
pinger_app = get_pinger_app('pinger')
relay_app = get_pinger_app('relay')
pinger_dns = dns_format.format('pinger', marathon_service_name)
relay_dns = dns_format.format('relay', marathon_service_name)
# test pinger app to master
shakedown.copy_file_to_master(fixture_dir() + "/pinger.py")
with shakedown.master_http_service():
# need to add app with http service in place or it will fail to fetch
client.add_app(pinger_app)
client.add_app(relay_app)
shakedown.deployment_wait()
shakedown.wait_for_dns(relay_dns)
relay_url = 'http://{}:7777/relay-ping?url={}:7777'.format(
relay_dns, pinger_dns
)
@retrying.retry
def http_output_check(stop_max_attempt_number=30):
status, output = shakedown.run_command_on_master('curl {}'.format(relay_url))
assert status
assert 'Pong /pinger' in output
assert 'Relay from /relay' in output
http_output_check()
def _pods_json(file="simple-pods.json"):
return get_resource(os.path.join(fixture_dir(), file))
def partition_agent(hostname):
"""Partition a node from all network traffic except for SSH and loopback"""
shakedown.copy_file_to_agent(hostname, "{}/net-services-agent.sh".format(fixture_dir()))
print("partitioning {}".format(hostname))
shakedown.run_command_on_agent(hostname, 'sh net-services-agent.sh fail')
def _pods_json(file="simple-pods.json"):
return get_resource(os.path.join(fixture_dir(), file))
