def __init__(self, container, network, host, vip=None, ipv6=False):
self._network = network
self._container = container
if network in [marathon.Network.HOST, marathon.Network.BRIDGE]:
# both of these cases will rely on marathon to assign ports
self.app, self.uuid = test_helpers.marathon_test_app(
network=network,
host_constraint=host,
vip=vip,
container_type=container,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
elif network == marathon.Network.USER:
self.app, self.uuid = test_helpers.marathon_test_app(
network=marathon.Network.USER,
network_name='dcos6' if ipv6 else 'dcos',
host_port=unused_port(),
host_constraint=host,
vip=vip,
container_type=container,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
if vip is not None and container == marathon.Container.DOCKER:
del self.app['container']['docker']['portMappings'][0]['hostPort']
# allow this app to run on public slaves
self.app['acceptedResourceRoles'] = ['*', 'slave_public']
self.id = self.app['id']
def __init__(self, container, network, host, vip=None, ipv6=False):
self._network = network
self._container = container
if network in [marathon.Network.HOST, marathon.Network.BRIDGE]:
# both of these cases will rely on marathon to assign ports
self.app, self.uuid = test_helpers.marathon_test_app(
network=network,
host_constraint=host,
vip=vip,
container_type=container,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
elif network == marathon.Network.USER:
self.app, self.uuid = test_helpers.marathon_test_app(
network=marathon.Network.USER,
network_name='dcos6' if ipv6 else 'dcos',
host_port=unused_port(),
host_constraint=host,
vip=vip,
container_type=container,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
if vip is not None and container == marathon.Container.DOCKER:
del self.app['container']['docker']['portMappings'][0][
'hostPort']
# allow this app to run on public slaves
self.app['acceptedResourceRoles'] = ['*', 'slave_public']
self.id = self.app['id']
def test_ip_per_container(dcos_api_session):
'''Test if we are able to connect to a task with ip-per-container mode
'''
# Launch the test_server in ip-per-container mode (user network)
if len(dcos_api_session.slaves) < 2:
pytest.skip("IP Per Container tests require 2 private agents to work")
app_definition, test_uuid = test_helpers.marathon_test_app(
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
container_type=marathon.Container.DOCKER,
network=marathon.Network.USER,
host_port=9080)
app_definition['instances'] = 2
app_definition['constraints'] = [['hostname', 'UNIQUE']]
with dcos_api_session.marathon.deploy_and_cleanup(app_definition,
check_health=True):
service_points = dcos_api_session.marathon.get_app_service_endpoints(
app_definition['id'])
app_port = app_definition['container']['portMappings'][0][
'containerPort']
cmd = '/opt/mesosphere/bin/curl -s -f -m 5 http://{}:{}/ping'.format(
service_points[1].ip, app_port)
ensure_routable(cmd, service_points[0].host, service_points[0].port)
def test_service_discovery_docker_overlay_port_mapping(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
network=marathon.Network.USER,
host_port=9080)
assert_service_discovery(dcos_api_session, app_definition, [DNSOverlay, DNSPortMap])
def test_service_discovery_mesos_overlay(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
network=marathon.Network.USER)
assert_service_discovery(dcos_api_session, app_definition, [DNSOverlay])
def __init__(self,
container,
network,
host=None,
vip=None,
network_name=None,
app_name_fmt=None,
host_port=None):
if host_port is None:
host_port = unused_port()
args = {
'app_name_fmt': app_name_fmt,
'network': network,
'host_port': host_port,
'vip': vip,
'container_type': container,
'healthcheck_protocol': marathon.Healthcheck.MESOS_HTTP
}
if host is not None:
args['host_constraint'] = host
if network == marathon.Network.USER:
args['container_port'] = unused_port()
if network_name is not None:
args['network_name'] = network_name
if vip is not None:
del args['host_port']
self.app, self.uuid = test_helpers.marathon_test_app(**args)
# allow this app to run on public slaves
self.app['acceptedResourceRoles'] = ['*', 'slave_public']
self.id = self.app['id']
def test_if_ucr_app_runs_in_new_pid_namespace(
dcos_api_session: DcosApiSession) -> None:
# We run a marathon app instead of a metronome job because metronome
# doesn't support running docker images with the UCR. We need this
# functionality in order to test that the pid namespace isolator
# is functioning correctly.
app, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS)
ps_output_file = 'ps_output'
app['cmd'] = 'ps ax -o pid= > {}; sleep 1000'.format(ps_output_file)
with dcos_api_session.marathon.deploy_and_cleanup(app, check_health=False):
marathon_framework_id = dcos_api_session.marathon.get(
'/v2/info').json()['frameworkId']
app_task = dcos_api_session.marathon.get('/v2/apps/{}/tasks'.format(
app['id'])).json()['tasks'][0]
# There is a short delay between the `app_task` starting and it writing
# its output to the `pd_output_file`. Because of this, we wait up to 10
# seconds for this file to appear before throwing an exception.
@retrying.retry(wait_fixed=1000, stop_max_delay=10000)
def get_ps_output() -> Any:
return dcos_api_session.mesos_sandbox_file(app_task['slaveId'],
marathon_framework_id,
app_task['id'],
ps_output_file)
assert len(
get_ps_output().split()
) <= 4, 'UCR app has more than 4 processes running in its pid namespace'
def test_service_discovery_docker_overlay(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.USER,
host_port=9080)
del app_definition['container']['docker']['portMappings'][0]['hostPort']
assert_service_discovery(dcos_api_session, app_definition, [DNSOverlay])
def test_service_discovery_docker_bridge(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.BRIDGE,
container_port=2020,
host_port=9080)
assert_service_discovery(dcos_api_session, app_definition, [DNSPortMap])
def test_service_discovery_mesos_overlay(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
network=marathon.Network.USER)
assert_service_discovery(dcos_api_session, app_definition, [DNSOverlay])
def octarine_runner(dcos_api_session, mode, uuid, uri, bind_port=None):
log.info("Running octarine(mode={}, uuid={}, uri={}".format(mode, uuid, uri))
octarine = "/opt/mesosphere/bin/octarine"
bind_port_str = ""
if bind_port is not None:
bind_port_str = "-bindPort {}".format(bind_port)
server_cmd = "{} -mode {} {} {}".format(octarine, mode, bind_port_str, uuid)
log.info("Server: {}".format(server_cmd))
proxy = ('http://127.0.0.1:$({} --client --port {})'.format(octarine, uuid))
curl_cmd = '''"$(curl --fail --proxy {} {})"'''.format(proxy, uri)
expected_output = '''"$(printf "{\\n \\"pong\\": true\\n}")"'''
check_cmd = """sh -c '[ {} = {} ]'""".format(curl_cmd, expected_output)
log.info("Check: {}".format(check_cmd))
app, uuid = test_helpers.marathon_test_app()
app['requirePorts'] = True
app['cmd'] = server_cmd
app['healthChecks'] = [{
"protocol": "COMMAND",
"command": {"value": check_cmd},
'gracePeriodSeconds': 5,
'intervalSeconds': 10,
'timeoutSeconds': 10,
'maxConsecutiveFailures': 30
}]
with dcos_api_session.marathon.deploy_and_cleanup(app):
pass
def test_service_discovery_docker_overlay_port_mapping(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.USER,
host_port=9080)
assert_service_discovery(dcos_api_session, app_definition,
[DNSOverlay, DNSPortMap])
def test_service_discovery_docker_overlay(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.USER,
host_port=9080)
del app_definition['container']['docker']['portMappings'][0]['hostPort']
assert_service_discovery(dcos_api_session, app_definition, [DNSOverlay])
def test_service_discovery_docker_bridge(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.BRIDGE,
container_port=2020,
host_port=9080)
assert_service_discovery(dcos_api_session, app_definition, [DNSPortMap])
def test_service_discovery_mesos_host(
dcos_api_session: DcosApiSession) -> None:
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS,
healthcheck_protocol=marathon.Healthcheck.HTTP)
assert_service_discovery(dcos_api_session, app_definition, [DNSHost])
def test_dcos_diagnostics_bundle_create_download_delete(
dcos_api_session: DcosApiSession, use_legacy_api: bool) -> None:
"""
test bundle create, read, delete workflow
"""
health_url = dcos_api_session.default_url.copy(
query='cache=0',
path='system/health/v1',
)
diagnostics = Diagnostics(
default_url=health_url,
masters=dcos_api_session.masters,
all_slaves=dcos_api_session.all_slaves,
session=dcos_api_session.copy().session,
use_legacy_api=use_legacy_api,
)
app, test_uuid = test_helpers.marathon_test_app()
with dcos_api_session.marathon.deploy_and_cleanup(app, timeout=120):
bundle = _create_bundle(diagnostics)
_check_diagnostics_bundle_status(dcos_api_session)
_download_and_extract_bundle(dcos_api_session, bundle, diagnostics)
_download_and_extract_bundle_from_another_master(
dcos_api_session, bundle, diagnostics)
_delete_bundle(diagnostics, bundle)
def test_octarine(dcos_api_session, timeout=30):
expanded_config = test_helpers.get_expanded_config()
if expanded_config.get('security') == 'strict':
pytest.skip('See: https://jira.mesosphere.com/browse/DCOS-14760')
# This app binds to port 80. This is only required by the http (not srv)
# transparent mode test. In transparent mode, we use ".mydcos.directory"
# to go to localhost, the port attached there is only used to
# determine which port to send traffic to on localhost. When it
# reaches the proxy, the port is not used, and a request is made
# to port 80.
app, uuid = test_helpers.marathon_test_app(host_port=80)
app['acceptedResourceRoles'] = ["slave_public"]
app['requirePorts'] = True
with dcos_api_session.marathon.deploy_and_cleanup(app):
service_points = dcos_api_session.marathon.get_app_service_endpoints(
app['id'])
port_number = service_points[0].port
# It didn't actually grab port 80 when requirePorts was unset
assert port_number == app['portDefinitions'][0]["port"]
app_name = app["id"].strip("/")
port_name = app['portDefinitions'][0]["name"]
port_protocol = app['portDefinitions'][0]["protocol"]
srv = "_{}._{}._{}.marathon.mesos".format(port_name, app_name,
port_protocol)
addr = "{}.marathon.mesos".format(app_name)
transparent_suffix = ".mydcos.directory"
standard_mode = "standard"
transparent_mode = "transparent"
t_addr_bind = 2508
t_srv_bind = 2509
standard_addr = "{}:{}/ping".format(addr, port_number)
standard_srv = "{}/ping".format(srv)
transparent_addr = "{}{}:{}/ping".format(addr, transparent_suffix,
t_addr_bind)
transparent_srv = "{}{}:{}/ping".format(srv, transparent_suffix,
t_srv_bind)
# The uuids are different between runs so that they don't have a
# chance of colliding. They shouldn't anyways, but just to be safe.
octarine_runner(dcos_api_session, standard_mode, uuid + "1",
standard_addr)
octarine_runner(dcos_api_session, standard_mode, uuid + "2",
standard_srv)
octarine_runner(dcos_api_session,
transparent_mode,
uuid + "3",
transparent_addr,
bind_port=t_addr_bind)
octarine_runner(dcos_api_session,
transparent_mode,
uuid + "4",
transparent_srv,
bind_port=t_srv_bind)
def test_service_discovery_docker_overlay_port_mapping(
dcos_api_session: DcosApiSession) -> None:
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
network=marathon.Network.USER,
host_port=9080)
assert_service_discovery(dcos_api_session, app_definition,
[DNSOverlay, DNSPortMap])
def vip_app(container: marathon.Container, network: marathon.Network, host: str, vip: str):
# user_net_port is only actually used for USER network because this cannot be assigned
# by marathon
if network in [marathon.Network.HOST, marathon.Network.BRIDGE]:
# both of these cases will rely on marathon to assign ports
return test_helpers.marathon_test_app(
network=network,
host_constraint=host,
vip=vip,
container_type=container)
elif network == marathon.Network.USER:
return test_helpers.marathon_test_app(
network=network,
host_port=unused_port(marathon.Network.USER),
host_constraint=host,
vip=vip,
container_type=container)
else:
raise AssertionError('Unexpected network: {}'.format(network.value))
def test_dcos_diagnostics_bundle_create_download_delete(dcos_api_session):
"""
test bundle create, read, delete workflow
"""
app, test_uuid = test_helpers.marathon_test_app()
with dcos_api_session.marathon.deploy_and_cleanup(app):
bundle = _create_bundle(dcos_api_session)
_check_diagnostics_bundle_status(dcos_api_session)
_download_and_extract_bundle(dcos_api_session, bundle)
_download_and_extract_bundle_from_another_master(dcos_api_session, bundle)
_delete_bundle(dcos_api_session, bundle)
def test_dcos_diagnostics_bundle_create_download_delete(dcos_api_session):
"""
test bundle create, read, delete workflow
"""
app, test_uuid = test_helpers.marathon_test_app()
with dcos_api_session.marathon.deploy_and_cleanup(app):
bundle = _create_bundle(dcos_api_session)
_check_diagnostics_bundle_status(dcos_api_session)
_download_and_extract_bundle(dcos_api_session, bundle)
_download_and_extract_bundle_from_another_master(dcos_api_session, bundle)
_delete_bundle(dcos_api_session, bundle)
def test_if_ucr_app_can_be_deployed(dcos_api_session, healthcheck):
"""Marathon app inside ucr deployment integration test.
Verifies that a marathon docker app inside of a ucr container can be
deployed and accessed as expected.
"""
deploy_test_app_and_check(
dcos_api_session,
*test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS,
healthcheck_protocol=healthcheck))
def test_if_docker_app_can_be_deployed(dcos_api_session):
"""Marathon app inside docker deployment integration test.
Verifies that a marathon app inside of a docker daemon container can be
deployed and accessed as expected.
"""
deploy_test_app_and_check(
dcos_api_session,
*test_helpers.marathon_test_app(
network=marathon.Network.BRIDGE,
container_type=marathon.Container.DOCKER,
container_port=9080))
def test_files_api(dcos_api_session):
app, test_uuid = test_helpers.marathon_test_app()
with dcos_api_session.marathon.deploy_and_cleanup(app):
marathon_framework_id = dcos_api_session.marathon.get('/v2/info').json()['frameworkId']
app_task = dcos_api_session.marathon.get('/v2/apps/{}/tasks'.format(app['id'])).json()['tasks'][0]
for required_sandbox_file in ('stdout', 'stderr'):
content = dcos_api_session.mesos_sandbox_file(
app_task['slaveId'], marathon_framework_id, app_task['id'], required_sandbox_file)
assert content, 'File {} should not be empty'.format(required_sandbox_file)
def test_l4lb(dcos_api_session):
'''Test l4lb is load balancing between all the backends
* create 5 apps using the same VIP
* get uuid from the VIP in parallel from many threads
* verify that 5 uuids have been returned
* only testing if all 5 are hit at least once
'''
if not lb_enabled():
pytest.skip('Load Balancer disabled')
numapps = 5
numthreads = numapps * 4
apps = []
rvs = deque()
backends = []
dnsname = 'l4lbtest.marathon.l4lb.thisdcos.directory:5000'
with contextlib.ExitStack() as stack:
for _ in range(numapps):
origin_app, origin_uuid = \
test_helpers.marathon_test_app(
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
# same vip for all the apps
origin_app['portDefinitions'][0]['labels'] = {'VIP_0': '/l4lbtest:5000'}
apps.append(origin_app)
stack.enter_context(dcos_api_session.marathon.deploy_and_cleanup(origin_app))
sp = dcos_api_session.marathon.get_app_service_endpoints(origin_app['id'])
backends.append({'port': sp[0].port, 'ip': sp[0].host})
# make sure that the service point responds
geturl('http://{}:{}/ping'.format(sp[0].host, sp[0].port))
# make sure that the VIP is responding too
geturl('http://{}/ping'.format(dnsname))
vips = geturl("http://localhost:62080/v1/vips")
[vip] = [vip for vip in vips if vip['vip'] == dnsname and vip['protocol'] == 'tcp']
for backend in vip['backend']:
backends.remove(backend)
assert backends == []
# do many requests in parallel.
def thread_request():
# deque is thread safe
rvs.append(geturl('http://l4lbtest.marathon.l4lb.thisdcos.directory:5000/test_uuid'))
threads = [threading.Thread(target=thread_request) for i in range(0, numthreads)]
for t in threads:
t.start()
for t in threads:
t.join()
expected_uuids = [a['id'].split('-')[2] for a in apps]
received_uuids = [r['test_uuid'] for r in rvs if r is not None]
assert len(set(expected_uuids)) == numapps
assert len(set(received_uuids)) == numapps
assert set(expected_uuids) == set(received_uuids)
def test_l4lb(dcos_api_session):
'''Test l4lb is load balancing between all the backends
* create 5 apps using the same VIP
* get uuid from the VIP in parallel from many threads
* verify that 5 uuids have been returned
* only testing if all 5 are hit at least once
'''
if not lb_enabled():
pytest.skip('Load Balancer disabled')
numapps = 5
numthreads = numapps * 4
apps = []
rvs = deque()
backends = []
dnsname = 'l4lbtest.marathon.l4lb.thisdcos.directory:5000'
with contextlib.ExitStack() as stack:
for _ in range(numapps):
origin_app, origin_uuid = \
test_helpers.marathon_test_app(
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
# same vip for all the apps
origin_app['portDefinitions'][0]['labels'] = {'VIP_0': '/l4lbtest:5000'}
apps.append(origin_app)
stack.enter_context(dcos_api_session.marathon.deploy_and_cleanup(origin_app))
sp = dcos_api_session.marathon.get_app_service_endpoints(origin_app['id'])
backends.append({'port': sp[0].port, 'ip': sp[0].host})
# make sure that the service point responds
geturl('http://{}:{}/ping'.format(sp[0].host, sp[0].port))
# make sure that the VIP is responding too
geturl('http://{}/ping'.format(dnsname))
vips = geturl("http://localhost:62080/v1/vips")
[vip] = [vip for vip in vips if vip['vip'] == dnsname and vip['protocol'] == 'tcp']
for backend in vip['backend']:
backends.remove(backend)
assert backends == []
# do many requests in parallel.
def thread_request():
# deque is thread safe
rvs.append(geturl('http://l4lbtest.marathon.l4lb.thisdcos.directory:5000/test_uuid'))
threads = [threading.Thread(target=thread_request) for i in range(0, numthreads)]
for t in threads:
t.start()
for t in threads:
t.join()
expected_uuids = [a['id'].split('-')[2] for a in apps]
received_uuids = [r['test_uuid'] for r in rvs if r is not None]
assert len(set(expected_uuids)) == numapps
assert len(set(received_uuids)) == numapps
assert set(expected_uuids) == set(received_uuids)
def __init__(self, container, network, host, vip=None):
self._network = network
self._container = container
if network in [marathon.Network.HOST, marathon.Network.BRIDGE]:
# both of these cases will rely on marathon to assign ports
self.app, self.uuid = test_helpers.marathon_test_app(
network=network,
host_constraint=host,
vip=vip,
container_type=container,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
elif network == marathon.Network.USER:
self.app, self.uuid = test_helpers.marathon_test_app(
network=network,
host_port=unused_port(),
host_constraint=host,
vip=vip,
container_type=container,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
# allow this app to run on public slaves
self.app['acceptedResourceRoles'] = ['*', 'slave_public']
self.id = self.app['id']
def __init__(self, container, network, host, vip=None):
self._network = network
self._container = container
if network in [marathon.Network.HOST, marathon.Network.BRIDGE]:
# both of these cases will rely on marathon to assign ports
self.app, self.uuid = test_helpers.marathon_test_app(
network=network,
host_constraint=host,
vip=vip,
container_type=container,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
elif network == marathon.Network.USER:
self.app, self.uuid = test_helpers.marathon_test_app(
network=network,
host_port=unused_port(),
host_constraint=host,
vip=vip,
container_type=container,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP)
# allow this app to run on public slaves
self.app['acceptedResourceRoles'] = ['*', 'slave_public']
self.id = self.app['id']
def test_if_marathon_app_can_be_deployed(dcos_api_session):
"""Marathon app deployment integration test
This test verifies that marathon app can be deployed, and that service points
returned by Marathon indeed point to the app that was deployed.
The application being deployed is a simple http server written in python.
Please test_server.py for more details.
This is done by assigning an unique UUID to each app and passing it to the
docker container as an env variable. After successful deployment, the
"GET /test_uuid" request is issued to the app. If the returned UUID matches
the one assigned to test - test succeeds.
"""
deploy_test_app_and_check(dcos_api_session, *test_helpers.marathon_test_app())
def test_if_marathon_app_can_be_deployed_with_mesos_containerizer(dcos_api_session):
"""Marathon app deployment integration test using the Mesos Containerizer
This test verifies that a Marathon app using the Mesos containerizer with
a Docker image can be deployed.
This is done by assigning an unique UUID to each app and passing it to the
docker container as an env variable. After successfull deployment, the
"GET /test_uuid" request is issued to the app. If the returned UUID matches
the one assigned to test - test succeds.
When port mapping is available (MESOS-4777), this test should be updated to
reflect that.
"""
deploy_test_app_and_check(
dcos_api_session,
*test_helpers.marathon_test_app(container_type=marathon.Container.MESOS))
def test_octarine(dcos_api_session, timeout=30):
expanded_config = test_helpers.get_expanded_config()
if expanded_config.get('security') == 'strict':
pytest.skip('See: https://jira.mesosphere.com/browse/DCOS-14760')
# This app binds to port 80. This is only required by the http (not srv)
# transparent mode test. In transparent mode, we use ".mydcos.directory"
# to go to localhost, the port attached there is only used to
# determine which port to send traffic to on localhost. When it
# reaches the proxy, the port is not used, and a request is made
# to port 80.
app, uuid = test_helpers.marathon_test_app(host_port=80)
app['acceptedResourceRoles'] = ["slave_public"]
app['requirePorts'] = True
with dcos_api_session.marathon.deploy_and_cleanup(app):
service_points = dcos_api_session.marathon.get_app_service_endpoints(app['id'])
port_number = service_points[0].port
# It didn't actually grab port 80 when requirePorts was unset
assert port_number == app['portDefinitions'][0]["port"]
app_name = app["id"].strip("/")
port_name = app['portDefinitions'][0]["name"]
port_protocol = app['portDefinitions'][0]["protocol"]
srv = "_{}._{}._{}.marathon.mesos".format(port_name, app_name, port_protocol)
addr = "{}.marathon.mesos".format(app_name)
transparent_suffix = ".mydcos.directory"
standard_mode = "standard"
transparent_mode = "transparent"
t_addr_bind = 2508
t_srv_bind = 2509
standard_addr = "{}:{}/ping".format(addr, port_number)
standard_srv = "{}/ping".format(srv)
transparent_addr = "{}{}:{}/ping".format(addr, transparent_suffix, t_addr_bind)
transparent_srv = "{}{}:{}/ping".format(srv, transparent_suffix, t_srv_bind)
# The uuids are different between runs so that they don't have a
# chance of colliding. They shouldn't anyways, but just to be safe.
octarine_runner(dcos_api_session, standard_mode, uuid + "1", standard_addr)
octarine_runner(dcos_api_session, standard_mode, uuid + "2", standard_srv)
octarine_runner(dcos_api_session, transparent_mode, uuid + "3", transparent_addr, bind_port=t_addr_bind)
octarine_runner(dcos_api_session, transparent_mode, uuid + "4", transparent_srv, bind_port=t_srv_bind)
def test_ip_per_container(dcos_api_session):
'''Test if we are able to connect to a task with ip-per-container mode
'''
# Launch the test_server in ip-per-container mode (user network)
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.USER,
host_port=9080)
assert len(dcos_api_session.slaves) >= 2, 'IP Per Container tests require 2 private agents to work'
app_definition['instances'] = 2
app_definition['constraints'] = [['hostname', 'UNIQUE']]
with dcos_api_session.marathon.deploy_and_cleanup(app_definition, check_health=True):
service_points = dcos_api_session.marathon.get_app_service_endpoints(app_definition['id'])
app_port = app_definition['container']['docker']['portMappings'][0]['containerPort']
cmd = '/opt/mesosphere/bin/curl -s -f -m 5 http://{}:{}/ping'.format(service_points[1].ip, app_port)
ensure_routable(cmd, service_points[0].host, service_points[0].port)
def test_files_api(dcos_api_session):
'''
This test verifies that the standard output and error of a Mesos task can be
read. We check that neither standard output nor error are empty files. Since
the default `marathon_test_app()` does not write to its standard output the
task definition is modified to output something there.
'''
app, test_uuid = test_helpers.marathon_test_app()
app['cmd'] = 'echo $DCOS_TEST_UUID && ' + app['cmd']
with dcos_api_session.marathon.deploy_and_cleanup(app):
marathon_framework_id = dcos_api_session.marathon.get('/v2/info').json()['frameworkId']
app_task = dcos_api_session.marathon.get('/v2/apps/{}/tasks'.format(app['id'])).json()['tasks'][0]
for required_sandbox_file in ('stdout', 'stderr'):
content = dcos_api_session.mesos_sandbox_file(
app_task['slaveId'], marathon_framework_id, app_task['id'], required_sandbox_file)
assert content, 'File {} should not be empty'.format(required_sandbox_file)
def __init__(self, container, network, host, vip=None, ipv6=False, app_name_fmt=None):
args = {
'app_name_fmt': app_name_fmt,
'network': network,
'host_port': unused_port(),
'host_constraint': host,
'vip': vip,
'container_type': container,
'healthcheck_protocol': marathon.Healthcheck.MESOS_HTTP
}
if network == marathon.Network.USER:
args['container_port'] = unused_port()
if ipv6:
args['network_name'] = 'dcos6'
if vip is not None:
del args['host_port']
self.app, self.uuid = test_helpers.marathon_test_app(**args)
# allow this app to run on public slaves
self.app['acceptedResourceRoles'] = ['*', 'slave_public']
self.id = self.app['id']
def test_if_search_is_working(dcos_api_session):
"""Test if custom set search is working.
Verifies that a marathon app running on the dcos_api_session can resolve names using
searching the "search" the dcos_api_session was launched with (if any). It also tests
that absolute searches still work, and search + things that aren't
sub-domains fails properly.
The application being deployed is a simple http server written in python.
Please check test_server.py for more details.
"""
# Launch the app
app_definition, test_uuid = test_helpers.marathon_test_app()
with dcos_api_session.marathon.deploy_and_cleanup(app_definition):
service_points = dcos_api_session.marathon.get_app_service_endpoints(
app_definition['id'])
# Get the status
r = requests.get('http://{}:{}/dns_search'.format(
service_points[0].host, service_points[0].port))
if r.status_code != 200:
msg = "Test server replied with non-200 reply: '{0} {1}. "
msg += "Detailed explanation of the problem: {2}"
pytest.fail(msg.format(r.status_code, r.reason, r.text))
r_data = r.json()
# Make sure we hit the app we expected
assert r_data['test_uuid'] == test_uuid
expected_error = {'error': '[Errno -2] Name or service not known'}
# Check that result matches expectations for this dcos_api_session
expanded_config = test_helpers.get_expanded_config()
if expanded_config['dns_search']:
assert r_data['search_hit_leader'] in dcos_api_session.masters
assert r_data['always_hit_leader'] in dcos_api_session.masters
assert r_data['always_miss'] == expected_error
else: # No dns search, search hit should miss.
assert r_data['search_hit_leader'] == expected_error
assert r_data['always_hit_leader'] in dcos_api_session.masters
assert r_data['always_miss'] == expected_error
def test_if_search_is_working(dcos_api_session):
"""Test if custom set search is working.
Verifies that a marathon app running on the dcos_api_session can resolve names using
searching the "search" the dcos_api_session was launched with (if any). It also tests
that absolute searches still work, and search + things that aren't
sub-domains fails properly.
The application being deployed is a simple http server written in python.
Please check test_server.py for more details.
"""
# Launch the app
app_definition, test_uuid = test_helpers.marathon_test_app()
with dcos_api_session.marathon.deploy_and_cleanup(app_definition):
service_points = dcos_api_session.marathon.get_app_service_endpoints(app_definition['id'])
# Get the status
r = requests.get('http://{}:{}/dns_search'.format(service_points[0].host,
service_points[0].port))
if r.status_code != 200:
msg = "Test server replied with non-200 reply: '{0} {1}. "
msg += "Detailed explanation of the problem: {2}"
pytest.fail(msg.format(r.status_code, r.reason, r.text))
r_data = r.json()
# Make sure we hit the app we expected
assert r_data['test_uuid'] == test_uuid
expected_error = {'error': '[Errno -2] Name or service not known'}
# Check that result matches expectations for this dcos_api_session
expanded_config = test_helpers.get_expanded_config()
if expanded_config['dns_search']:
assert r_data['search_hit_leader'] in dcos_api_session.masters
assert r_data['always_hit_leader'] in dcos_api_session.masters
assert r_data['always_miss'] == expected_error
else: # No dns search, search hit should miss.
assert r_data['search_hit_leader'] == expected_error
assert r_data['always_hit_leader'] in dcos_api_session.masters
assert r_data['always_miss'] == expected_error
def test_if_ucr_app_runs_in_new_pid_namespace(dcos_api_session):
# We run a marathon app instead of a metronome job because metronome
# doesn't support running docker images with the UCR. We need this
# functionality in order to test that the pid namespace isolator
# is functioning correctly.
app, test_uuid = test_helpers.marathon_test_app(container_type=marathon.Container.MESOS)
ps_output_file = 'ps_output'
app['cmd'] = 'ps ax -o pid= > {}; sleep 1000'.format(ps_output_file)
with dcos_api_session.marathon.deploy_and_cleanup(app, check_health=False):
marathon_framework_id = dcos_api_session.marathon.get('/v2/info').json()['frameworkId']
app_task = dcos_api_session.marathon.get('/v2/apps/{}/tasks'.format(app['id'])).json()['tasks'][0]
# There is a short delay between the `app_task` starting and it writing
# its output to the `pd_output_file`. Because of this, we wait up to 10
# seconds for this file to appear before throwing an exception.
@retrying.retry(wait_fixed=1000, stop_max_delay=10000)
def get_ps_output():
return dcos_api_session.mesos_sandbox_file(
app_task['slaveId'], marathon_framework_id, app_task['id'], ps_output_file)
assert len(get_ps_output().split()) <= 4, 'UCR app has more than 4 processes running in its pid namespace'
def test_dcos_cni_l4lb(dcos_api_session):
'''
This tests the `dcos - l4lb` CNI plugins:
https: // github.com / dcos / dcos - cni / tree / master / cmd / l4lb
The `dcos-l4lb` CNI plugins allows containers running on networks that don't
necessarily have routes to spartan interfaces and minuteman VIPs to consume DNS
service from spartan and layer-4 load-balancing services from minuteman by
injecting spartan and minuteman services into the container's network
namespace. You can read more about the motivation for this CNI plugin and type
of problems it solves in this design doc:
https://docs.google.com/document/d/1xxvkFknC56hF-EcDmZ9tzKsGiZdGKBUPfrPKYs85j1k/edit?usp=sharing
In order to test `dcos-l4lb` CNI plugin we emulate a virtual network that
lacks routes for spartan interface and minuteman VIPs. In this test, we
first install a virtual network called `spartan-net` on one of the agents.
The `spartan-net` is a CNI network that is a simple BRIDGE network with the
caveat that it doesn't have any default routes. `spartan-net` has routes
only for the agent network. In other words it doesn't have any routes
towards the spartan-interfaces or minuteman VIPs.
We then run a server (our python ping-pong server) on the DC/OS overlay.
Finally to test that the `dcos-l4lb` plugin, which is also part of
`spartan-net` is able to inject the Minuteman and Spartan services into the
contianer's netns, we start a client on the `spartan-net` and try to `curl` the
`ping-pong` server using its VIP. Without the Minuteman and Spartan services
injected in the container's netns the expectation would be that this `curl`
would fail, with a successful `curl` execution on the VIP allowing the
test-case to PASS.
'''
# CNI configuration of `spartan-net`.
spartan_net = {
'cniVersion': '0.2.0',
'name': 'spartan-net',
'type': 'dcos-l4lb',
'delegate': {
'type': 'mesos-cni-port-mapper',
'excludeDevices': ['sprt-cni0'],
'chain': 'spartan-net',
'delegate': {
'type': 'bridge',
'bridge': 'sprt-cni0',
'ipMasq': True,
'isGateway': True,
'ipam': {
'type': 'host-local',
'subnet': '192.168.250.0/24',
'routes': [
# Reachability to DC/OS overlay.
{'dst': '9.0.0.0/8'},
# Reachability to all private address subnet. We need
# this reachability since different cloud providers use
# different private address spaces to launch tenant
# networks.
{'dst': '10.0.0.0/8'},
{'dst': '172.16.0.0/12'},
{'dst': '192.168.0.0/16'}
]
}
}
}
}
log.info("spartan-net config:{}".format(json.dumps(spartan_net)))
# Application to deploy CNI configuration.
cni_config_app, config_uuid = test_helpers.marathon_test_app()
# Override the default test app command with a command to write the CNI
# configuration.
#
# NOTE: We add the sleep at the end of this command so that the task stays
# alive for the test harness to make sure that the task got deployed.
# Ideally we should be able to deploy one of tasks using the test harness
# but that doesn't seem to be the case here.
cni_config_app['cmd'] = 'echo \'{}\' > /opt/mesosphere/etc/dcos/network/cni/spartan.cni && sleep 10000'.format(
json.dumps(spartan_net))
log.info("App for setting CNI config: {}".format(json.dumps(cni_config_app)))
try:
dcos_api_session.marathon.deploy_app(cni_config_app, check_health=False)
except Exception as ex:
raise AssertionError("Couldn't install CNI config for `spartan-net`".format(json.dumps(cni_config_app))) from ex
# Get the host on which the `spartan-net` was installed.
cni_config_app_service = None
try:
cni_config_app_service = dcos_api_session.marathon.get_app_service_endpoints(cni_config_app['id'])
except Exception as ex:
raise AssertionError("Couldn't retrieve the host on which `spartan-net` was installed.") from ex
# We only have one instance of `cni_config_app_service`.
spartan_net_host = cni_config_app_service[0].host
# Launch the test-app on DC/OS overlay, with a VIP.
server_vip_port = unused_port()
server_vip = '/spartanvip:{}'.format(server_vip_port)
server_vip_addr = 'spartanvip.marathon.l4lb.thisdcos.directory:{}'.format(server_vip_port)
# Launch the test_server in ip-per-container mode (user network)
server, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
network=marathon.Network.USER,
host_port=9080,
vip=server_vip)
# Launch the server on the DC/OS overlay
server['ipAddress']['networkName'] = 'dcos'
log.info("Launching server with VIP:{} on network {}".format(server_vip_addr, server['ipAddress']['networkName']))
try:
dcos_api_session.marathon.deploy_app(server, check_health=False)
except Exception as ex:
raise AssertionError(
"Couldn't launch server on 'dcos':{}".format(server['ipAddress']['networkName'])) from ex
# Get the client app on the 'spartan-net' network.
#
# NOTE: Currently, we are creating the app-def by hand instead of relying
# on the harness to create this app-def since the marathon harness does not
# allow any port-mapping for CNI networks at this point.
client_port = 9081
client, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
network=marathon.Network.USER,
host_port=client_port,
vip=server_vip,
host_constraint=spartan_net_host)
client["container"]["portMappings"] = [
{
'containerPort': client_port,
'hostPort': client_port,
'protocol': 'tcp',
'name': 'http'
}
]
# Remove the `ipAddress` entry for DC/OS overlay.
del client["ipAddress"]
# Attach this container to the `spartan-net` network. We are using the v2
# network API here.
client["networks"] = [
{
"mode": "container",
"name": "spartan-net"
}
]
try:
dcos_api_session.marathon.deploy_app(client, check_health=False)
except Exception as ex:
raise AssertionError("Couldn't launch client on 'spartan-net':{}".format(client)) from ex
# Change the client command task to do a curl on the server we just deployed.
cmd = '/opt/mesosphere/bin/curl -s -f -m 5 http://{}/ping'.format(server_vip_addr)
try:
response = ensure_routable(cmd, spartan_net_host, client_port)
log.info("Received a response from {}: {}".format(server_vip_addr, response))
except Exception as ex:
raise AssertionError("Unable to query VIP: {}".format(server_vip_addr)) from ex
def test_app_networking_mode_with_defined_container_port(
dcos_api_session, networking_mode, host_port):
"""
The Admin Router can proxy a request on the `/service/[app]`
endpoint to an application running in a container in different networking
modes with manually or automatically assigned host port on which is
the application HTTP endpoint exposed.
Networking modes are testing following configurations:
- host
- container
- container/bridge
https://mesosphere.github.io/marathon/docs/networking.html#networking-modes
"""
app_definition, test_uuid = test_helpers.marathon_test_app(
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
container_type=marathon.Container.DOCKER,
network=networking_mode,
host_port=host_port)
dcos_service_name = uuid.uuid4().hex
app_definition['labels'] = {
'DCOS_SERVICE_NAME': dcos_service_name,
'DCOS_SERVICE_PORT_INDEX': '0',
'DCOS_SERVICE_SCHEME': 'http',
}
# Arbitrary buffer time, accounting for propagation/processing delay.
buffer_time = 5
# Cache refresh in Adminrouter takes 30 seconds at most.
# CACHE_POLL_PERIOD=25s + valid=5s Nginx resolver DNS entry TTL
# https://github.com/dcos/dcos/blob/cb9105ee537cc44cbe63cc7c53b3b01b764703a0/
# packages/adminrouter/extra/src/includes/http/master.conf#L21
adminrouter_default_refresh = 25 + 5 + buffer_time
app_id = app_definition['id']
app_instances = app_definition['instances']
app_definition['constraints'] = [['hostname', 'UNIQUE']]
# For the routing check to work, two conditions must be true:
#
# 1. The application must be deployed, so that `/ping` responds with 200.
# 2. The Admin Router routing layer must not be using an outdated
# version of the Nginx resolver cache.
#
# We therefore wait until these conditions have certainly been met.
# We wait for the Admin Router cache refresh first so that there is
# unlikely to be much double-waiting. That is, we do not want to be waiting
# for the cache to refresh when it already refreshed while we were waiting
# for the app to become healthy.
with dcos_api_session.marathon.deploy_and_cleanup(app_definition,
check_health=False):
time.sleep(adminrouter_default_refresh)
dcos_api_session.marathon.wait_for_app_deployment(
app_id=app_id,
app_instances=app_instances,
check_health=False,
ignore_failed_tasks=False,
timeout=1200,
)
r = dcos_api_session.get('/service/' + dcos_service_name + '/ping')
assert r.status_code == 200
assert 'pong' in r.json()
def test_app_networking_mode_with_defined_container_port(dcos_api_session, networking_mode, host_port):
"""
The Admin Router can proxy a request on the `/service/[app]`
endpoint to an application running in a container in different networking
modes with manually or automatically assigned host port on which is
the application HTTP endpoint exposed.
Networking modes are testing following configurations:
- host
- container
- container/bridge
https://mesosphere.github.io/marathon/docs/networking.html#networking-modes
"""
app_definition, test_uuid = test_helpers.marathon_test_app(
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
container_type=marathon.Container.DOCKER,
network=networking_mode,
host_port=host_port)
dcos_service_name = uuid.uuid4().hex
app_definition['labels'] = {
'DCOS_SERVICE_NAME': dcos_service_name,
'DCOS_SERVICE_PORT_INDEX': '0',
'DCOS_SERVICE_SCHEME': 'http',
}
# Arbitrary buffer time, accounting for propagation/processing delay.
buffer_time = 5
# Cache refresh in Adminrouter takes 30 seconds at most.
# CACHE_POLL_PERIOD=25s + valid=5s Nginx resolver DNS entry TTL
# https://github.com/dcos/dcos/blob/cb9105ee537cc44cbe63cc7c53b3b01b764703a0/
# packages/adminrouter/extra/src/includes/http/master.conf#L21
adminrouter_default_refresh = 25 + 5 + buffer_time
app_id = app_definition['id']
app_instances = app_definition['instances']
app_definition['constraints'] = [['hostname', 'UNIQUE']]
# For the routing check to work, two conditions must be true:
#
# 1. The application must be deployed, so that `/ping` responds with 200.
# 2. The Admin Router routing layer must not be using an outdated
# version of the Nginx resolver cache.
#
# We therefore wait until these conditions have certainly been met.
# We wait for the Admin Router cache refresh first so that there is
# unlikely to be much double-waiting. That is, we do not want to be waiting
# for the cache to refresh when it already refreshed while we were waiting
# for the app to become healthy.
with dcos_api_session.marathon.deploy_and_cleanup(app_definition, check_health=False):
time.sleep(adminrouter_default_refresh)
dcos_api_session.marathon.wait_for_app_deployment(
app_id=app_id,
app_instances=app_instances,
check_health=False,
ignore_failed_tasks=False,
timeout=1200,
)
r = dcos_api_session.get('/service/' + dcos_service_name + '/ping')
assert r.status_code == 200
assert 'pong' in r.json()
def test_dcos_cni_l4lb(dcos_api_session):
'''
This tests the `dcos - l4lb` CNI plugins:
https: // github.com / dcos / dcos - cni / tree / master / cmd / l4lb
The `dcos-l4lb` CNI plugins allows containers running on networks that don't
necessarily have routes to spartan interfaces and minuteman VIPs to consume DNS
service from spartan and layer-4 load-balancing services from minuteman by
injecting spartan and minuteman services into the container's network
namespace. You can read more about the motivation for this CNI plugin and type
of problems it solves in this design doc:
https://docs.google.com/document/d/1xxvkFknC56hF-EcDmZ9tzKsGiZdGKBUPfrPKYs85j1k/edit?usp=sharing
In order to test `dcos-l4lb` CNI plugin we emulate a virtual network that
lacks routes for spartan interface and minuteman VIPs. In this test, we
first install a virtual network called `spartan-net` on one of the agents.
The `spartan-net` is a CNI network that is a simple BRIDGE network with the
caveat that it doesn't have any default routes. `spartan-net` has routes
only for the agent network. In other words it doesn't have any routes
towards the spartan-interfaces or minuteman VIPs.
We then run a server (our python ping-pong server) on the DC/OS overlay.
Finally to test that the `dcos-l4lb` plugin, which is also part of
`spartan-net` is able to inject the Minuteman and Spartan services into the
contianer's netns, we start a client on the `spartan-net` and try to `curl` the
`ping-pong` server using its VIP. Without the Minuteman and Spartan services
injected in the container's netns the expectation would be that this `curl`
would fail, with a successful `curl` execution on the VIP allowing the
test-case to PASS.
'''
# CNI configuration of `spartan-net`.
spartan_net = {
'cniVersion': '0.2.0',
'name': 'spartan-net',
'type': 'dcos-l4lb',
'delegate': {
'type': 'mesos-cni-port-mapper',
'excludeDevices': ['sprt-cni0'],
'chain': 'spartan-net',
'delegate': {
'type': 'bridge',
'bridge': 'sprt-cni0',
'ipMasq': True,
'isGateway': True,
'ipam': {
'type':
'host-local',
'subnet':
'192.168.250.0/24',
'routes': [
# Reachability to DC/OS overlay.
{
'dst': '9.0.0.0/8'
},
# Reachability to all private address subnet. We need
# this reachability since different cloud providers use
# different private address spaces to launch tenant
# networks.
{
'dst': '10.0.0.0/8'
},
{
'dst': '172.16.0.0/12'
},
{
'dst': '192.168.0.0/16'
}
]
}
}
}
}
log.info("spartan-net config:{}".format(json.dumps(spartan_net)))
# Application to deploy CNI configuration.
cni_config_app, config_uuid = test_helpers.marathon_test_app()
# Override the default test app command with a command to write the CNI
# configuration.
#
# NOTE: We add the sleep at the end of this command so that the task stays
# alive for the test harness to make sure that the task got deployed.
# Ideally we should be able to deploy one of tasks using the test harness
# but that doesn't seem to be the case here.
cni_config_app[
'cmd'] = 'echo \'{}\' > /opt/mesosphere/etc/dcos/network/cni/spartan.cni && sleep 10000'.format(
json.dumps(spartan_net))
log.info("App for setting CNI config: {}".format(
json.dumps(cni_config_app)))
try:
dcos_api_session.marathon.deploy_app(cni_config_app,
check_health=False)
except Exception as ex:
raise AssertionError(
"Couldn't install CNI config for `spartan-net`".format(
json.dumps(cni_config_app))) from ex
# Get the host on which the `spartan-net` was installed.
cni_config_app_service = None
try:
cni_config_app_service = dcos_api_session.marathon.get_app_service_endpoints(
cni_config_app['id'])
except Exception as ex:
raise AssertionError(
"Couldn't retrieve the host on which `spartan-net` was installed."
) from ex
# We only have one instance of `cni_config_app_service`.
spartan_net_host = cni_config_app_service[0].host
# Launch the test-app on DC/OS overlay, with a VIP.
server_vip_port = unused_port()
server_vip = '/spartanvip:{}'.format(server_vip_port)
server_vip_addr = 'spartanvip.marathon.l4lb.thisdcos.directory:{}'.format(
server_vip_port)
# Launch the test_server in ip-per-container mode (user network)
server, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
network=marathon.Network.USER,
host_port=9080,
vip=server_vip)
# Launch the server on the DC/OS overlay
server['ipAddress']['networkName'] = 'dcos'
log.info("Launching server with VIP:{} on network {}".format(
server_vip_addr, server['ipAddress']['networkName']))
try:
dcos_api_session.marathon.deploy_app(server, check_health=False)
except Exception as ex:
raise AssertionError("Couldn't launch server on 'dcos':{}".format(
server['ipAddress']['networkName'])) from ex
# Get the client app on the 'spartan-net' network.
#
# NOTE: Currently, we are creating the app-def by hand instead of relying
# on the harness to create this app-def since the marathon harness does not
# allow any port-mapping for CNI networks at this point.
client_port = 9081
client, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS,
healthcheck_protocol=marathon.Healthcheck.MESOS_HTTP,
network=marathon.Network.USER,
host_port=client_port,
vip=server_vip,
host_constraint=spartan_net_host)
client["container"]["portMappings"] = [{
'containerPort': client_port,
'hostPort': client_port,
'protocol': 'tcp',
'name': 'http'
}]
# Remove the `ipAddress` entry for DC/OS overlay.
del client["ipAddress"]
# Attach this container to the `spartan-net` network. We are using the v2
# network API here.
client["networks"] = [{"mode": "container", "name": "spartan-net"}]
try:
dcos_api_session.marathon.deploy_app(client, check_health=False)
except Exception as ex:
raise AssertionError(
"Couldn't launch client on 'spartan-net':{}".format(
client)) from ex
# Change the client command task to do a curl on the server we just deployed.
cmd = '/opt/mesosphere/bin/curl -s -f -m 5 http://{}/ping'.format(
server_vip_addr)
try:
response = ensure_routable(cmd, spartan_net_host, client_port)
log.info("Received a response from {}: {}".format(
server_vip_addr, response))
except Exception as ex:
raise AssertionError(
"Unable to query VIP: {}".format(server_vip_addr)) from ex
def _service_discovery_test(dcos_api_session, docker_network_bridge):
"""Service discovery integration test
This test verifies if service discovery works, by comparing marathon data
with information from mesos-dns and from containers themselves.
This is achieved by deploying an application to marathon with two instances
, and ["hostname", "UNIQUE"] constraint set. This should result in containers
being deployed to two different slaves.
The application being deployed is a simple http server written in python.
Please check test_server.py for more details.
Next thing is comparing the service points provided by marathon with those
reported by mesos-dns. The tricky part here is that may take some time for
mesos-dns to catch up with changes in the dcos_api_session.
And finally, one of service points is verified in as-seen-by-other-containers
fashion.
+------------------------+ +------------------------+
| Slave 1 | | Slave 2 |
| | | |
| +--------------------+ | | +--------------------+ |
+--------------+ | | | | | | | |
| | | | App instance A +------>+ App instance B | |
| TC Agent +<---->+ | | | | | |
| | | | "test server" +<------+ "reflector" | |
+--------------+ | | | | | | | |
| +--------------------+ | | +--------------------+ |
+------------------------+ +------------------------+
Code running on TC agent connects to one of the containers (let's call it
"test server") and makes a POST request with IP and PORT service point of
the second container as parameters (let's call it "reflector"). The test
server in turn connects to other container and makes a "GET /reflect"
request. The reflector responds with test server's IP as seen by it and
the session UUID as provided to it by Marathon. This data is then returned
to TC agent in response to POST request issued earlier.
The test succeeds if test UUIDs of the test server, reflector and the test
itself match and the IP of the test server matches the service point of that
container as reported by Marathon.
"""
# TODO(cmaloney): For non docker network bridge we should just do a mesos container.
if docker_network_bridge:
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.BRIDGE,
container_port=2020,
host_port=9080)
else:
app_definition, test_uuid = test_helpers.marathon_test_app(container_type=marathon.Container.DOCKER)
app_definition['instances'] = 2
assert len(dcos_api_session.slaves) >= 2, "Test requires a minimum of two agents"
app_definition["constraints"] = [["hostname", "UNIQUE"], ]
with dcos_api_session.marathon.deploy_and_cleanup(app_definition):
service_points = dcos_api_session.marathon.get_app_service_endpoints(app_definition['id'])
# Verify if Mesos-DNS agrees with Marathon:
@retrying.retry(wait_fixed=1000,
stop_max_delay=DNS_ENTRY_UPDATE_TIMEOUT * 1000,
retry_on_result=lambda ret: ret is None,
retry_on_exception=lambda x: False)
def _pool_for_mesos_dns():
r = dcos_api_session.get('/mesos_dns/v1/services/_{}._tcp.marathon.mesos'.format(
app_definition['id'].lstrip('/')))
assert r.status_code == 200
r_data = r.json()
if r_data == [{'host': '', 'port': '', 'service': '', 'ip': ''}] or len(r_data) < len(service_points):
logging.info("Waiting for Mesos-DNS to update entries")
return None
else:
logging.info("Mesos-DNS entries have been updated!")
return r_data
try:
r_data = _pool_for_mesos_dns()
except retrying.RetryError:
msg = "Mesos DNS has failed to update entries in {} seconds."
pytest.fail(msg.format(DNS_ENTRY_UPDATE_TIMEOUT))
marathon_provided_servicepoints = sorted((x.host, x.port) for x in service_points)
mesosdns_provided_servicepoints = sorted((x['ip'], int(x['port'])) for x in r_data)
assert marathon_provided_servicepoints == mesosdns_provided_servicepoints
# Verify if containers themselves confirm what Marathon says:
payload = {"reflector_ip": service_points[1].host,
"reflector_port": service_points[1].port}
r = requests.post('http://{}:{}/your_ip'.format(
service_points[0].host, service_points[0].port), payload)
if r.status_code != 200:
msg = "Test server replied with non-200 reply: '{status_code} {reason}. "
msg += "Detailed explanation of the problem: {text}"
pytest.fail(msg.format(status_code=r.status_code, reason=r.reason, text=r.text))
r_data = r.json()
assert r_data['reflector_uuid'] == test_uuid
assert r_data['test_uuid'] == test_uuid
if len(dcos_api_session.slaves) >= 2:
# When len(slaves)==1, we are connecting through docker-proxy using
# docker0 interface ip. This makes this assertion useless, so we skip
# it and rely on matching test uuid between containers only.
assert r_data['my_ip'] == service_points[0].host
def test_if_marathon_app_can_be_debugged(dcos_api_session):
# Launch a basic marathon app (no image), so we can debug into it!
# Cannot use deploy_and_cleanup because we must attach to a running app/task/container.
app, test_uuid = test_helpers.marathon_test_app()
app_id = 'integration-test-{}'.format(test_uuid)
with dcos_api_session.marathon.deploy_and_cleanup(app):
# Fetch the mesos master state once the task is running
master_ip = dcos_api_session.masters[0]
r = dcos_api_session.get('/state', host=master_ip, port=5050)
assert r.status_code == 200
state = r.json()
# Find the agent_id and container_id from master state
container_id = None
agent_id = None
for framework in state['frameworks']:
for task in framework['tasks']:
if app_id in task['id']:
container_id = task['statuses'][0]['container_status']['container_id']['value']
agent_id = task['slave_id']
assert container_id is not None, 'Container ID not found for instance of app_id {}'.format(app_id)
assert agent_id is not None, 'Agent ID not found for instance of app_id {}'.format(app_id)
# Find hostname and URL from agent_id
agent_hostname = None
for agent in state['slaves']:
if agent['id'] == agent_id:
agent_hostname = agent['hostname']
assert agent_hostname is not None, 'Agent hostname not found for agent_id {}'.format(agent_id)
logging.debug('Located %s with containerID %s on agent %s', app_id, container_id, agent_hostname)
def _post_agent(url, headers, json=None, data=None, stream=False):
r = dcos_api_session.post(
url,
host=agent_hostname,
port=5051,
headers=headers,
json=json,
data=data,
stream=stream)
assert r.status_code == 200
return r
# Prepare nested container id data
nested_container_id = {
'value': 'debug-%s' % str(uuid.uuid4()),
'parent': {'value': '%s' % container_id}}
# Launch debug session and attach to output stream of debug container
output_headers = {
'Content-Type': 'application/json',
'Accept': 'application/recordio',
'Message-Accept': 'application/json'
}
lncs_data = {
'type': 'LAUNCH_NESTED_CONTAINER_SESSION',
'launch_nested_container_session': {
'command': {'value': 'cat'},
'container_id': nested_container_id}}
launch_output = _post_agent('/api/v1', output_headers, json=lncs_data, stream=True)
# Attach to output stream of nested container
attach_out_data = {
'type': 'ATTACH_CONTAINER_OUTPUT',
'attach_container_output': {'container_id': nested_container_id}}
attached_output = _post_agent('/api/v1', output_headers, json=attach_out_data, stream=True)
# Attach to input stream of debug container and stream a message
input_headers = {
'Content-Type': 'application/recordio',
'Message-Content-Type': 'application/json',
'Accept': 'application/json',
'Transfer-Encoding': 'chunked'
}
_post_agent('/api/v1', input_headers, data=input_streamer(nested_container_id))
# Verify the streamed output from the launch session
meowed = False
decoder = recordio.Decoder(lambda s: json.loads(s.decode("UTF-8")))
for chunk in launch_output.iter_content():
for r in decoder.decode(chunk):
if r['type'] == 'DATA':
logging.debug('Extracted data chunk: %s', r['data'])
assert r['data']['data'] == 'meow', 'Output did not match expected'
meowed = True
assert meowed, 'Read launch output without seeing meow.'
meowed = False
# Verify the message from the attached output stream
for chunk in attached_output.iter_content():
for r in decoder.decode(chunk):
if r['type'] == 'DATA':
logging.debug('Extracted data chunk: %s', r['data'])
assert r['data']['data'] == 'meow', 'Output did not match expected'
meowed = True
assert meowed, 'Read output stream without seeing meow.'
def test_blkio_stats(dcos_api_session):
# Launch a Marathon application to do some disk writes, and then verify that
# the cgroups blkio statistics of the application can be correctly retrieved.
app, test_uuid = test_helpers.marathon_test_app(container_type=marathon.Container.MESOS)
app_id = 'integration-test-{}'.format(test_uuid)
# The application will generate a 10k file with 10 disk writes.
#
# TODO(qianzhang): In some old platforms (CentOS 6 and Ubuntu 14),
# the first disk write of a blkio cgroup will always be missed in
# the blkio throttling statistics, so here we run two `dd` commands,
# the first one which does only one disk write will be missed on
# those platforms, and the second one will be recorded in the blkio
# throttling statistics. When we drop the CentOS 6 and Ubuntu 14
# support in future, we should remove the first `dd` command.
marker_file = 'marker'
app['cmd'] = ('dd if=/dev/zero of=file bs=1024 count=1 oflag=dsync && '
'dd if=/dev/zero of=file bs=1024 count=10 oflag=dsync && '
'echo -n done > {} && sleep 1000').format(marker_file)
with dcos_api_session.marathon.deploy_and_cleanup(app, check_health=False):
marathon_framework_id = dcos_api_session.marathon.get('/v2/info').json()['frameworkId']
app_task = dcos_api_session.marathon.get('/v2/apps/{}/tasks'.format(app['id'])).json()['tasks'][0]
# Wait up to 10 seconds for the marker file to appear which
# indicates the disk writes via `dd` command are done.
@retrying.retry(wait_fixed=1000, stop_max_delay=10000)
def get_marker_file_content():
return dcos_api_session.mesos_sandbox_file(
app_task['slaveId'], marathon_framework_id, app_task['id'], marker_file)
assert get_marker_file_content() == 'done'
# Fetch the Mesos master state
master_ip = dcos_api_session.masters[0]
r = dcos_api_session.get('/state', host=master_ip, port=5050)
assert r.status_code == 200
state = r.json()
# Find the agent_id from master state
agent_id = None
for framework in state['frameworks']:
for task in framework['tasks']:
if app_id in task['id']:
agent_id = task['slave_id']
assert agent_id is not None, 'Agent ID not found for instance of app_id {}'.format(app_id)
# Find hostname from agent_id
agent_hostname = None
for agent in state['slaves']:
if agent['id'] == agent_id:
agent_hostname = agent['hostname']
assert agent_hostname is not None, 'Agent hostname not found for agent_id {}'.format(agent_id)
logging.debug('Located %s on agent %s', app_id, agent_hostname)
# Fetch the Mesos agent statistics
r = dcos_api_session.get('/monitor/statistics', host=agent_hostname, port=5051)
assert r.status_code == 200
stats = r.json()
total_io_serviced = None
total_io_service_bytes = None
for stat in stats:
# Find the statistic for the Marathon application that we deployed. Since what that
# Marathon application launched is a Mesos command task (i.e., using Mesos built-in
# command executor), the executor ID will be same as the task ID, so if we find the
# `app_id` in an executor ID of a statistic, that must be the statistic entry
# corresponding to the application that we deployed.
if app_id in stat['executor_id']:
# We only care about the blkio throttle statistics but not the blkio cfq statistics,
# because in the environment where the disk IO scheduler is not `cfq`, all the cfq
# statistics may be 0.
throttle_stats = stat['statistics']['blkio_statistics']['throttling']
for throttle_stat in throttle_stats:
if 'device' not in throttle_stat:
total_io_serviced = throttle_stat['io_serviced'][0]['value']
total_io_service_bytes = throttle_stat['io_service_bytes'][0]['value']
assert total_io_serviced is not None, ('Total blkio throttling IO serviced not found '
'for app_id {}'.format(app_id))
assert total_io_service_bytes is not None, ('Total blkio throttling IO service bytes '
'not found for app_id {}'.format(app_id))
# We expect the statistics retrieved from Mesos agent are equal or greater than what we
# did with the `dd` command (i.e., 10 and 10240), because:
# 1. Besides the disk writes done by the `dd` command, the statistics may also include
# some disk reads, e.g., to load the necessary executable binary and libraries.
# 2. In the environment where RAID is enabled, there may be multiple disk writes to
# different disks for a single `dd` write.
assert int(total_io_serviced) >= 10, ('Total blkio throttling IO serviced for app_id {} '
'are less than 10'.format(app_id))
assert int(total_io_service_bytes) >= 10240, ('Total blkio throttling IO service bytes for '
'app_id {} are less than 10240'.format(app_id))
def test_service_discovery_mesos_host(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.MESOS, healthcheck_protocol=marathon.Healthcheck.HTTP)
assert_service_discovery(dcos_api_session, app_definition, [DNSHost])
def test_service_discovery_docker_host(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.HOST)
assert_service_discovery(dcos_api_session, app_definition, [DNSHost])
def _service_discovery_test(dcos_api_session, docker_network_bridge):
"""Service discovery integration test
This test verifies if service discovery works, by comparing marathon data
with information from mesos-dns and from containers themselves.
This is achieved by deploying an application to marathon with two instances
, and ["hostname", "UNIQUE"] constraint set. This should result in containers
being deployed to two different slaves.
The application being deployed is a simple http server written in python.
Please check test_server.py for more details.
Next thing is comparing the service points provided by marathon with those
reported by mesos-dns. The tricky part here is that may take some time for
mesos-dns to catch up with changes in the dcos_api_session.
And finally, one of service points is verified in as-seen-by-other-containers
fashion.
+------------------------+ +------------------------+
| Slave 1 | | Slave 2 |
| | | |
| +--------------------+ | | +--------------------+ |
+--------------+ | | | | | | | |
| | | | App instance A +------>+ App instance B | |
| TC Agent +<---->+ | | | | | |
| | | | "test server" +<------+ "reflector" | |
+--------------+ | | | | | | | |
| +--------------------+ | | +--------------------+ |
+------------------------+ +------------------------+
Code running on TC agent connects to one of the containers (let's call it
"test server") and makes a POST request with IP and PORT service point of
the second container as parameters (let's call it "reflector"). The test
server in turn connects to other container and makes a "GET /reflect"
request. The reflector responds with test server's IP as seen by it and
the session UUID as provided to it by Marathon. This data is then returned
to TC agent in response to POST request issued earlier.
The test succeeds if test UUIDs of the test server, reflector and the test
itself match and the IP of the test server matches the service point of that
container as reported by Marathon.
"""
# TODO(cmaloney): For non docker network bridge we should just do a mesos container.
if docker_network_bridge:
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.BRIDGE,
container_port=2020,
host_port=9080)
else:
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER)
app_definition['instances'] = 2
if len(dcos_api_session.slaves) < 2:
pytest.skip("Service Discovery Tests require a minimum of two agents.")
app_definition["constraints"] = [
["hostname", "UNIQUE"],
]
with dcos_api_session.marathon.deploy_and_cleanup(app_definition):
service_points = dcos_api_session.marathon.get_app_service_endpoints(
app_definition['id'])
# Verify if Mesos-DNS agrees with Marathon:
@retrying.retry(wait_fixed=1000,
stop_max_delay=DNS_ENTRY_UPDATE_TIMEOUT * 1000,
retry_on_result=lambda ret: ret is None,
retry_on_exception=lambda x: False)
def _pool_for_mesos_dns():
r = dcos_api_session.get(
'/mesos_dns/v1/services/_{}._tcp.marathon.mesos'.format(
app_definition['id'].lstrip('/')))
assert r.status_code == 200
r_data = r.json()
if r_data == [{
'host': '',
'port': '',
'service': '',
'ip': ''
}] or len(r_data) < len(service_points):
logging.info("Waiting for Mesos-DNS to update entries")
return None
else:
logging.info("Mesos-DNS entries have been updated!")
return r_data
try:
r_data = _pool_for_mesos_dns()
except retrying.RetryError:
msg = "Mesos DNS has failed to update entries in {} seconds."
pytest.fail(msg.format(DNS_ENTRY_UPDATE_TIMEOUT))
marathon_provided_servicepoints = sorted(
(x.host, x.port) for x in service_points)
mesosdns_provided_servicepoints = sorted(
(x['ip'], int(x['port'])) for x in r_data)
assert marathon_provided_servicepoints == mesosdns_provided_servicepoints
# Verify if containers themselves confirm what Marathon says:
payload = {
"reflector_ip": service_points[1].host,
"reflector_port": service_points[1].port
}
r = requests.post(
'http://{}:{}/your_ip'.format(service_points[0].host,
service_points[0].port), payload)
if r.status_code != 200:
msg = "Test server replied with non-200 reply: '{status_code} {reason}. "
msg += "Detailed explanation of the problem: {text}"
pytest.fail(
msg.format(status_code=r.status_code,
reason=r.reason,
text=r.text))
r_data = r.json()
assert r_data['reflector_uuid'] == test_uuid
assert r_data['test_uuid'] == test_uuid
if len(dcos_api_session.slaves) >= 2:
# When len(slaves)==1, we are connecting through docker-proxy using
# docker0 interface ip. This makes this assertion useless, so we skip
# it and rely on matching test uuid between containers only.
assert r_data['my_ip'] == service_points[0].host
def test_service_discovery_docker_overlay(dcos_api_session):
app_definition, test_uuid = test_helpers.marathon_test_app(
container_type=marathon.Container.DOCKER,
network=marathon.Network.USER)
assert_service_discovery(dcos_api_session, app_definition, [DNSOverlay])