def uninstall(service_name, package_name=None):
start = time.time()
if package_name is None:
package_name = service_name
print('Uninstalling/janitoring {}'.format(service_name))
try:
shakedown.uninstall_package_and_wait(package_name, service_name=service_name)
except (dcos.errors.DCOSException, ValueError) as e:
print('Got exception when uninstalling package, ' +
'continuing with janitor anyway: {}'.format(e))
janitor_start = time.time()
janitor_cmd = (
'docker run mesosphere/janitor /janitor.py '
'-r {svc}-role -p {svc}-principal -z dcos-service-{svc} --auth_token={auth}')
shakedown.run_command_on_master(janitor_cmd.format(
svc=service_name,
auth=shakedown.run_dcos_command('config show core.dcos_acs_token')[0].strip()))
finish = time.time()
print('Uninstall done after pkg({}) + janitor({}) = total({})'.format(
sdk_spin.pretty_time(janitor_start - start),
sdk_spin.pretty_time(finish - janitor_start),
sdk_spin.pretty_time(finish - start)))
def test_launch_container_with_persistent_volume():
""" Tests launching a task with PV. It will write to a file in the PV.
The app is killed and restarted and we can still read from the PV.
"""
app_def = persistent_volume_app()
app_id = app_def['id']
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait()
tasks = client.get_tasks(app_id)
assert len(tasks) == 1
port = tasks[0]['ports'][0]
host = tasks[0]['host']
cmd = "curl {}:{}/data/foo".format(host, port)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert data == 'hello\n', "'{}' was not equal to hello\\n".format(data)
client.restart_app(app_id)
shakedown.deployment_wait()
tasks = client.get_tasks(app_id)
assert len(tasks) == 1
port = tasks[0]['ports'][0]
host = tasks[0]['host']
cmd = "curl {}:{}/data/foo".format(host, port)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert data == 'hello\nhello\n', "'{}' was not equal to hello\\nhello\\n".format(data)
def uninstall():
print("Uninstalling/janitoring {}".format(PACKAGE_NAME))
try:
shakedown.uninstall_package_and_wait(PACKAGE_NAME, service_name=PACKAGE_NAME)
except (dcos.errors.DCOSException, ValueError) as e:
print("Got exception when uninstalling package, continuing with janitor anyway: {}".format(e))
shakedown.run_command_on_master(
"docker run mesosphere/janitor /janitor.py "
"-r cassandra-role -p {} -z dcos-service-cassandra "
"--auth_token={}".format(PRINCIPAL, shakedown.run_dcos_command("config show core.dcos_acs_token")[0].strip())
)
def get_master():
exit_status, output = shakedown.run_command_on_master(
"{}/_cat/master'".format(_curl_api(service_name, "GET")))
if exit_status and len(output.split()) > 0:
return output.split()[-1]
return False
def uninstall():
print('Uninstalling/janitoring {}'.format(PACKAGE_NAME))
try:
shakedown.uninstall_package_and_wait(PACKAGE_NAME, app_id=PACKAGE_NAME)
except (dcos.errors.DCOSException, ValueError) as e:
print('Got exception when uninstalling package, continuing with janitor anyway: {}'.format(e))
shakedown.run_command_on_master(
'docker run mesosphere/janitor /janitor.py '
'-r cassandra-role -p cassandra-principal -z dcos-service-cassandra '
'--auth_token={}'.format(
shakedown.run_dcos_command(
'config show core.dcos_acs_token'
)[0].strip()
)
)
def assert_http_code(url, http_code='200'):
cmd = r'curl -s -o /dev/null -w "%{http_code}"'
cmd = cmd + ' {}'.format(url)
status, output = shakedown.run_command_on_master(cmd)
assert status
assert output == http_code
def test_pod_file_based_secret(secret_fixture):
secret_name, secret_value = secret_fixture
secret_normalized_name = secret_name.replace('/', '')
pod_id = '/{}'.format(uuid.uuid4().hex)
pod_def = {
"id": pod_id,
"containers": [{
"name": "container-1",
"resources": {
"cpus": 0.1,
"mem": 64
},
"endpoints": [{
"name": "http",
"hostPort": 0,
"protocol": [
"tcp"
]}
],
"exec": {
"command": {
"shell": "cat {} >> {}_file && /opt/mesosphere/bin/python -m http.server $ENDPOINT_HTTP".format(
secret_normalized_name, secret_normalized_name),
}
},
"volumeMounts": [{
"name": "vol",
"mountPath": secret_name
}],
}],
"networks": [{
"mode": "host"
}],
"volumes": [{
"name": "vol",
"secret": "secret1"
}],
"secrets": {
"secret1": {
"source": secret_name
}
}
}
client = marathon.create_client()
client.add_pod(pod_def)
shakedown.deployment_wait()
instances = client.show_pod(pod_id)['instances']
assert len(instances) == 1, 'Failed to start the file based secret pod'
port = instances[0]['containers'][0]['endpoints'][0]['allocatedHostPort']
host = instances[0]['networks'][0]['addresses'][0]
cmd = "curl {}:{}/{}_file".format(host, port, secret_normalized_name)
status, data = shakedown.run_command_on_master(cmd)
assert status, "{} did not succeed".format(cmd)
assert data.rstrip() == secret_value, "Got an unexpected secret data"
def test_docker_dns_mapping(marathon_service_name):
""" Tests that a running docker task is accessible from DNS.
"""
app_id = uuid.uuid4().hex
client = marathon.create_client()
app_json = app_docker(app_id)
client.add_app(app_json)
shakedown.deployment_wait()
tasks = client.get_tasks(app_id)
host = tasks[0]['host']
bad_cmd = 'ping -c 1 docker-test.marathon-user.mesos-bad'
status, output = shakedown.run_command_on_master(bad_cmd)
assert not status
@retrying.retry(stop_max_attempt_number=30)
def check_dns():
cmd = 'ping -c 1 {}.{}.mesos'.format(app_id, marathon_service_name)
shakedown.wait_for_dns('{}.{}.mesos'.format(app_id, marathon_service_name))
status, output = shakedown.run_command_on_master(cmd)
assert status
check_dns()
def test_mom_with_network_failure_bounce_master():
"""Marathon on Marathon (MoM) tests for DC/OS with network failures simulated by knocking out ports."""
# get MoM ip
mom_ip = common.ip_of_mom()
print("MoM IP: {}".format(mom_ip))
app_def = apps.sleep_app()
app_id = app_def["id"]
with shakedown.marathon_on_marathon():
client = marathon.create_client()
client.add_app(app_def)
shakedown.wait_for_task("marathon-user", app_id.lstrip('/'))
tasks = client.get_tasks(app_id)
original_task_id = tasks[0]["id"]
task_ip = tasks[0]['host']
print("\nTask IP: " + task_ip)
# PR for network partitioning in shakedown makes this better
# take out the net
partition_agent(mom_ip)
partition_agent(task_ip)
# wait for a min
time.sleep(timedelta(minutes=1).total_seconds())
# bounce master
shakedown.run_command_on_master("sudo systemctl restart dcos-mesos-master")
# bring the net up
reconnect_agent(mom_ip)
reconnect_agent(task_ip)
time.sleep(timedelta(minutes=1).total_seconds())
shakedown.wait_for_service_endpoint('marathon-user', timedelta(minutes=10).total_seconds())
with shakedown.marathon_on_marathon():
client = marathon.create_client()
shakedown.wait_for_task("marathon-user", app_id.lstrip('/'), timedelta(minutes=10).total_seconds())
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_task_is_back():
tasks = client.get_tasks(app_id)
assert tasks[0]['id'] == original_task_id, "The task ID has changed"
check_task_is_back()
def test_mom_with_network_failure_bounce_master():
"""Marathon on Marathon (MoM) tests for DC/OS with network failures simulated by
knocking out ports
"""
# get MoM ip
mom_ip = ip_of_mom()
print("MoM IP: {}".format(mom_ip))
app_def = get_resource("{}/large-sleep.json".format(fixture_dir()))
with shakedown.marathon_on_marathon():
client = marathon.create_client()
client.add_app(app_def)
shakedown.wait_for_task("marathon-user", "sleep")
tasks = client.get_tasks('sleep')
original_sleep_task_id = tasks[0]["id"]
task_ip = tasks[0]['host']
print("\nTask IP: " + task_ip)
# PR for network partitioning in shakedown makes this better
# take out the net
partition_agent(mom_ip)
partition_agent(task_ip)
# wait for a min
time.sleep(timedelta(minutes=1).total_seconds())
# bounce master
shakedown.run_command_on_master("sudo systemctl restart dcos-mesos-master")
# bring the net up
reconnect_agent(mom_ip)
reconnect_agent(task_ip)
time.sleep(timedelta(minutes=1).total_seconds())
shakedown.wait_for_service_endpoint('marathon-user')
shakedown.wait_for_task("marathon-user", "sleep")
with shakedown.marathon_on_marathon():
client = marathon.create_client()
shakedown.wait_for_task("marathon-user", "sleep")
tasks = client.get_tasks('sleep')
current_sleep_task_id = tasks[0]["id"]
assert current_sleep_task_id == original_sleep_task_id, "Task ID shouldn't change"
def run_hdfs_command(service_name, command):
"""
Execute the command using the Docker client
"""
full_command = 'docker run -e HDFS_SERVICE_NAME={} mesosphere/hdfs-client:2.6.4 /bin/bash -c "/configure-hdfs.sh && {}"'.format(service_name, command)
rc, output = shakedown.run_command_on_master(full_command)
return rc, output
def test_pod_secret_env_var(secret_fixture):
# Install enterprise-cli since it's needed to create secrets
if not common.is_enterprise_cli_package_installed():
common.install_enterprise_cli_package()
secret_name, secret_value = secret_fixture
pod_id = '/{}'.format(uuid.uuid4().hex)
pod_def = {
"id": pod_id,
"containers": [{
"name": "container-1",
"resources": {
"cpus": 0.1,
"mem": 64
},
"endpoints": [{
"name": "http",
"hostPort": 0,
"protocol": [
"tcp"
]}
],
"exec": {
"command": {
"shell": "echo $SECRET_ENV && echo $SECRET_ENV >> $MESOS_SANDBOX/secret-env && /opt/mesosphere/bin/python -m http.server $ENDPOINT_HTTP"
}
}
}],
"environment": {
"SECRET_ENV": {
"secret": "secret1"
}
},
"networks": [{
"mode": "host"
}],
"secrets": {
"secret1": {
"source": secret_name
}
}
}
client = marathon.create_client()
client.add_pod(pod_def)
shakedown.deployment_wait()
instances = client.show_pod(pod_id)['instances']
assert len(instances) == 1, 'Failed to start the secret environment variable pod'
port = instances[0]['containers'][0]['endpoints'][0]['allocatedHostPort']
host = instances[0]['networks'][0]['addresses'][0]
cmd = "curl {}:{}/secret-env".format(host, port)
status, data = shakedown.run_command_on_master(cmd)
assert status, "{} did not succeed".format(cmd)
assert data.rstrip() == secret_value
def master_ssh(cmd: str) -> tuple:
'''
Runs the provided command on the cluster master, using ssh.
Returns a boolean (==success) and a string (output)
'''
log.info('(SSH:master) {}'.format(cmd))
success, output = shakedown.run_command_on_master(cmd)
log.info('Output (success={}):\n{}'.format(success, output))
return success, output
def get_framework_srv_records(package_name):
cmd = "curl localhost:8123/v1/enumerate"
ok, out = shakedown.run_command_on_master(cmd)
assert ok, "Failed to get srv records. command was {}".format(cmd)
srvs = json.loads(out)
framework_srvs = [f for f in srvs["frameworks"] if f["name"] == package_name]
assert len(framework_srvs) == 1, "Got too many srv records matching package {}, got {}"\
.format(package_name, framework_srvs)
return framework_srvs[0]
def ee_version():
version = "NA"
# cat /opt/mesosphere/etc/bootstrap-config.json | jq '.["security"]'
status, stdout = run_command_on_master('cat /opt/mesosphere/etc/bootstrap-config.json')
if status:
configuration = json.loads(stdout)
version = configuration['security']
return version
def fn():
try:
shakedown.uninstall_package_and_wait(PACKAGE_NAME)
except (dcos.errors.DCOSException, json.decoder.JSONDecodeError):
return False
return shakedown.run_command_on_master(
'docker run mesosphere/janitor /janitor.py '
'-r cassandra-role -p cassandra-principal -z cassandra'
)
def fn():
command = (
"sudo kill -9 "
"$(ps ax | grep {} | grep -v grep | tr -s ' ' | sed 's/^ *//g' | "
"cut -d ' ' -f 1)".format(pattern))
if agent_host is None:
exit_status, _ = shakedown.run_command_on_master(command)
else:
exit_status, _ = shakedown.run_command_on_agent(agent_host, command)
return exit_status
def test_app_file_based_secret(secret_fixture):
# Install enterprise-cli since it's needed to create secrets
# if not common.is_enterprise_cli_package_installed():
# common.install_enterprise_cli_package()
secret_name, secret_value = secret_fixture
secret_normalized_name = secret_name.replace('/', '')
secret_container_path = 'mysecretpath'
app_id = uuid.uuid4().hex
# In case you're wondering about the `cmd`: secrets are mounted via tmpfs inside
# the container and are not visible outside, hence the intermediate file
app_def = {
"id": app_id,
"instances": 1,
"cpus": 0.1,
"mem": 64,
"cmd": "cat {} >> {}_file && /opt/mesosphere/bin/python -m http.server $PORT_API".
format(secret_container_path, secret_container_path),
"container": {
"type": "MESOS",
"volumes": [{
"containerPath": secret_container_path,
"secret": "secret1"
}]
},
"portDefinitions": [{
"port": 0,
"protocol": "tcp",
"name": "api",
"labels": {}
}],
"secrets": {
"secret1": {
"source": secret_name
}
}
}
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait()
tasks = client.get_tasks(app_id)
assert len(tasks) == 1, 'Failed to start the file based secret app'
port = tasks[0]['ports'][0]
host = tasks[0]['host']
# The secret by default is saved in $MESOS_SANDBOX/.secrets/path/to/secret
cmd = "curl {}:{}/{}_file".format(host, port, secret_container_path)
status, data = shakedown.run_command_on_master(cmd)
assert status, "{} did not succeed".format(cmd)
assert data == secret_value
def test_restart_container_with_persistent_volume():
"""A task with a persistent volume, which writes to a file in the persistent volume, is launched.
The app is killed and restarted and we can still read from the persistent volume what was written to it.
"""
app_def = apps.persistent_volume_app()
app_id = app_def['id']
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait()
tasks = client.get_tasks(app_id)
assert len(tasks) == 1, "The number of tasks is {} after deployment, but 1 was expected".format(len(tasks))
port = tasks[0]['ports'][0]
host = tasks[0]['host']
cmd = "curl {}:{}/data/foo".format(host, port)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert data == 'hello\n', "'{}' was not equal to hello\\n".format(data)
client.restart_app(app_id)
shakedown.deployment_wait()
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_task_recovery():
tasks = client.get_tasks(app_id)
assert len(tasks) == 1, "The number of tasks is {} after recovery, but 1 was expected".format(len(tasks))
check_task_recovery()
port = tasks[0]['ports'][0]
host = tasks[0]['host']
cmd = "curl {}:{}/data/foo".format(host, port)
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert data == 'hello\nhello\n', "'{}' was not equal to hello\\nhello\\n".format(data)
def kill_task_with_pattern(pattern, host=None):
command = (
"sudo kill -9 "
"$(ps ax | grep {} | grep -v grep | tr -s ' ' | sed 's/^ *//g' | "
"cut -d ' ' -f 1)".format(pattern))
if host is None:
result = shakedown.run_command_on_master(command)
else:
result = shakedown.run_command_on_agent(host, command)
if not result:
raise RuntimeError('Failed to kill task with pattern "{}"'.format(pattern))
def test_failing_health_check(static_port_config):
broker_id = '0'
broker_name = 'broker-' + broker_id
def found_broker(result):
return result != None, 'Broker not found.'
def broker_killed_result_checker(result):
return result, 'Broker not killed.'
print('Waiting for last Running Broker.')
test_utils.spin(get_running_broker_task_id, found_broker, 'broker-2')
# Get broker-0's task ID so we can know when it kills itself after failing
# the health check.
task_id = get_running_broker_task_id(broker_name)
print("{}'s task_id is {}".format(broker_name, task_id))
# Delete the ZK node which should trigger the health check to kill broker-0
shakedown.run_command_on_master(
'wget https://github.com/outbrain/zookeepercli/releases/'
'download/v1.0.10/zookeepercli'
)
shakedown.run_command_on_master('sudo chmod +x zookeepercli')
shakedown.run_command_on_master(
'./zookeepercli --servers 127.0.0.1 -c delete '
'/dcos-service-kafka/brokers/ids/' + broker_id
)
print('Waiting for Broker to fail.')
test_utils.spin(broker_killed, broker_killed_result_checker, task_id)
print('Waiting for Running Broker.')
test_utils.spin(get_running_broker_task_id, found_broker, broker_name)
def events_to_file():
print("entering events_to_file fixture")
shakedown.run_command_on_master('rm events.txt')
shakedown.run_command_on_master(
'curl --compressed -H "Cache-Control: no-cache" -H "Accept: text/event-stream" '
'-o events.txt leader.mesos:8080/v2/events &')
yield
shakedown.kill_process_on_host(shakedown.master_ip(), '[c]url')
shakedown.run_command_on_master('rm events.txt')
print("exiting events_to_file fixture")
def test_app_secret_env_var(secret_fixture):
# Install enterprise-cli since it's needed to create secrets
if not common.is_enterprise_cli_package_installed():
common.install_enterprise_cli_package()
secret_name, secret_value = secret_fixture
app_id = uuid.uuid4().hex
app_def = {
"id": app_id,
"instances": 1,
"cpus": 0.1,
"mem": 64,
"cmd": "echo $SECRET_ENV >> $MESOS_SANDBOX/secret-env && /opt/mesosphere/bin/python -m http.server $PORT_API",
"env": {
"SECRET_ENV": {
"secret": "secret1"
}
},
"portDefinitions": [{
"port": 0,
"protocol": "tcp",
"name": "api",
"labels": {}
}],
"secrets": {
"secret1": {
"source": secret_name
}
}
}
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait()
tasks = client.get_tasks(app_id)
assert len(tasks) == 1, 'Failed to start the secret environment variable app'
port = tasks[0]['ports'][0]
host = tasks[0]['host']
cmd = "curl {}:{}/secret-env".format(host, port)
status, data = shakedown.run_command_on_master(cmd)
assert status, "{} did not succeed".format(cmd)
assert data.rstrip() == secret_value
def test_marathon_backup_and_restore_leader(marathon_service_name):
backup_file = 'backup.tar'
backup_dir = '/tmp'
backup_url = 'file://{}/{}'.format(backup_dir, backup_file)
# Deploy a simple test app. It is expected to be there after leader reelection
client = marathon.create_client()
app_def = {
"id": "/sleep",
"instances": 1,
"cpus": 0.01,
"mem": 32,
"cmd": "sleep 100000"
}
app_id = app_def['id']
client.add_app(app_def)
shakedown.deployment_wait()
app = client.get_app(app_id)
assert app['tasksRunning'] == 1
task_id = app['tasks'][0]['id']
# Abdicate the leader with backup and restore
original_leader = shakedown.marathon_leader_ip()
print('leader: {}'.format(original_leader))
url = 'v2/leader?backup={}&restore={}'.format(backup_url, backup_url)
print('DELETE {}'.format(url))
common.delete_marathon_path(url)
# Wait for new leader (but same master server) to be up and ready
shakedown.wait_for_service_endpoint(marathon_service_name, timedelta(minutes=5).total_seconds())
app = client.get_app(app_id)
assert app['tasksRunning'] == 1
assert task_id == app['tasks'][0]['id'], "Task has a different Id after restore"
# Check if the backup file exits and is valid
cmd = 'tar -tf {}/{} | wc -l'.format(backup_dir, backup_file)
status, data = shakedown.run_command_on_master(cmd)
assert status, 'Failed to validate backup file {}'.format(backup_url)
assert int(data.rstrip()) > 0, "Backup file is empty"
def test_marathon_backup_and_restore_leader(marathon_service_name):
"""Backup and restore meeting is done with only one master since new master has to be able
to read the backup file that was created by the previous master and the easiest way to
test it is when there is 1 master
"""
backup_file = 'backup.tar'
backup_dir = '/tmp'
backup_url = 'file://{}/{}'.format(backup_dir, backup_file)
# Deploy a simple test app. It is expected to be there after leader reelection
app_def = apps.sleep_app()
app_id = app_def['id']
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait()
app = client.get_app(app_id)
assert app['tasksRunning'] == 1, "The number of running tasks is {}, but 1 was expected".format(app["tasksRunning"])
task_id = app['tasks'][0]['id']
# Abdicate the leader with backup and restore
original_leader = shakedown.marathon_leader_ip()
print('leader: {}'.format(original_leader))
url = 'v2/leader?backup={}&restore={}'.format(backup_url, backup_url)
print('DELETE {}'.format(url))
common.delete_marathon_path(url)
# Wait for new leader (but same master server) to be up and ready
shakedown.wait_for_service_endpoint(marathon_service_name, timedelta(minutes=5).total_seconds())
app = client.get_app(app_id)
assert app['tasksRunning'] == 1, "The number of running tasks is {}, but 1 was expected".format(app["tasksRunning"])
assert task_id == app['tasks'][0]['id'], "Task has a different ID after restore"
# Check if the backup file exits and is valid
cmd = 'tar -tf {}/{} | wc -l'.format(backup_dir, backup_file)
status, data = shakedown.run_command_on_master(cmd)
assert status, 'Failed to validate backup file {}'.format(backup_url)
assert int(data.rstrip()) > 0, "Backup file is empty"
def test_docker_dns_mapping(marathon_service_name):
"""Tests that a running Docker task is accessible via DNS."""
app_def = apps.docker_http_server()
client = marathon.create_client()
client.add_app(app_def)
shakedown.deployment_wait(app_id=app_def["id"])
bad_cmd = 'ping -c 1 docker-test.marathon-user.mesos-bad'
status, output = shakedown.run_command_on_master(bad_cmd)
assert not status
@retrying.retry(wait_fixed=1000, stop_max_attempt_number=30, retry_on_exception=common.ignore_exception)
def check_dns():
dnsname = '{}.{}.mesos'.format(app_def["id"].lstrip('/'), marathon_service_name)
cmd = 'ping -c 1 {}'.format(dnsname)
shakedown.wait_for_dns(dnsname)
status, output = shakedown.run_command_on_master(cmd)
assert status, "ping failed for app using DNS lookup: {}".format(dnsname)
check_dns()
def get_document(index_name, index_type, doc_id, service_name=PACKAGE_NAME):
exit_status, output = shakedown.run_command_on_master(
"{}/{}/{}/{}'".format(_curl_api(service_name, "GET"), index_name,
index_type, doc_id))
return output
def get_xpack_license(service_name=PACKAGE_NAME):
exit_status, output = shakedown.run_command_on_master(
"{}/_xpack/license'".format(_curl_api(service_name, "GET")))
return output
def delete_index(index_name, service_name=PACKAGE_NAME):
exit_status, output = shakedown.run_command_on_master("{}/{}'".format(
_curl_api(service_name, "DELETE"), index_name))
return output
def delete_index(index_name):
exit_status, output = shakedown.run_command_on_master("{}/{}'".format(
curl_api("DELETE"), index_name))
return output
def graph_api(index_name, query, service_name=PACKAGE_NAME):
exit_status, output = shakedown.run_command_on_master(
"{}/{}/_graph/explore' -d '{}'".format(_curl_api(service_name, "POST"),
index_name, json.dumps(query)))
return output
def check_kill_message():
status, stdout = shakedown.run_command_on_master('cat test.txt')
assert 'KILLED' in stdout
def http_output_check():
status, output = shakedown.run_command_on_master(
'curl {}'.format(relay_url))
assert status
assert 'Pong {}'.format(pinger_app["id"]) in output
assert 'Relay from {}'.format(relay_app["id"]) in output
def create_index(index_name, params):
command = "{}/{}' -d '{}'".format(curl_api("PUT"), index_name,
json.dumps(params))
exit_status, output = shakedown.run_command_on_master(command)
return output
def _run_janitor():
janitor_cmd = (
'docker run mesosphere/janitor /janitor.py '
'-r spark-role -p spark-principal -z spark_mesos_dispatcher --auth_token={auth}')
shakedown.run_command_on_master(janitor_cmd.format(
auth=shakedown.dcos_acs_token()))
def check_update_message():
status, stdout = shakedown.run_command_on_master('cat events.txt')
assert 'pod_updated_event' in stdout, 'pod_update_event event has not been produced'
def check_update_message():
status, stdout = shakedown.run_command_on_master('cat test.txt')
assert 'pod_updated_event' in stdout
def check_task(cmd, target_data):
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert target_data in data, "'{}' not found in {}".format(
target_data, data)
def graph_api(index_name, query):
command = "{}/{}/_graph/explore' -d '{}'".format(curl_api("POST"),
index_name,
json.dumps(query))
exit_status, output = shakedown.run_command_on_master(command)
return output
def systemctl_master(command='restart'):
shakedown.run_command_on_master('sudo systemctl {} dcos-mesos-master'.format(command))
def fun():
exit_status, output = shakedown.run_command_on_master(curl_cmd)
return output and "HTTP/1.1 200" in output
def get_document(index_name, index_type, doc_id):
exit_status, output = shakedown.run_command_on_master(
"{}/{}/{}/{}'".format(curl_api("GET"), index_name, index_type, doc_id))
return output
def check_deployment_message():
status, stdout = shakedown.run_command_on_master('cat test.txt')
assert 'event_stream_attached' in stdout
assert 'deployment_info' in stdout
assert 'deployment_step_success' in stdout
def get_document(index_name, index_type, doc_id):
exit_status, output = shakedown.run_command_on_master(
"{}/{}/{}/{}'".format(curl_api("GET"), index_name, index_type, doc_id))
return output
def test_pod_file_based_secret(secret_fixture):
# Install enterprise-cli since it's needed to create secrets
if not common.is_enterprise_cli_package_installed():
common.install_enterprise_cli_package()
secret_name, secret_value = secret_fixture
secret_normalized_name = secret_name.replace('/', '')
pod_id = '/{}'.format(uuid.uuid4().hex)
pod_def = {
"id":
pod_id,
"containers": [{
"name":
"container-1",
"resources": {
"cpus": 0.1,
"mem": 64
},
"endpoints": [{
"name": "http",
"hostPort": 0,
"protocol": ["tcp"]
}],
"exec": {
"command": {
"shell":
"cat {} >> {}_file && /opt/mesosphere/bin/python -m http.server $ENDPOINT_HTTP"
.format(secret_normalized_name, secret_normalized_name),
}
},
"volumeMounts": [{
"name": "vol",
"mountPath": secret_name
}],
}],
"networks": [{
"mode": "host"
}],
"volumes": [{
"name": "vol",
"secret": "secret1"
}],
"secrets": {
"secret1": {
"source": secret_name
}
}
}
client = marathon.create_client()
client.add_pod(pod_def)
shakedown.deployment_wait()
instances = client.show_pod(pod_id)['instances']
assert len(instances) == 1, 'Failed to start the file based secret pod'
port = instances[0]['containers'][0]['endpoints'][0]['allocatedHostPort']
host = instances[0]['networks'][0]['addresses'][0]
cmd = "curl {}:{}/{}_file".format(host, port, secret_normalized_name)
status, data = shakedown.run_command_on_master(cmd)
assert status, "{} did not succeed".format(cmd)
assert data.rstrip() == secret_value
def get_kibana_status(path):
token = shakedown.authenticate('bootstrapuser', 'deleteme')
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}{}".format(
token, shakedown.dcos_url(), path)
exit_status, output = shakedown.run_command_on_master(curl_cmd)
return output
def uninstall(service_name,
package_name=None,
role=None,
principal=None,
zk=None):
start = time.time()
if package_name is None:
package_name = service_name
if shakedown.dcos_version_less_than("1.10"):
sdk_utils.out('Uninstalling/janitoring {}'.format(service_name))
try:
shakedown.uninstall_package_and_wait(package_name,
service_name=service_name)
except (dcos.errors.DCOSException, ValueError) as e:
sdk_utils.out('Got exception when uninstalling package, ' +
'continuing with janitor anyway: {}'.format(e))
janitor_start = time.time()
# leading slash removed, other slashes converted to double underscores:
deslashed_service_name = service_name.lstrip('/').replace('/', '__')
if role is None:
role = deslashed_service_name + '-role'
if principal is None:
principal = service_name + '-principal'
if zk is None:
zk = 'dcos-service-' + deslashed_service_name
janitor_cmd = ('docker run mesosphere/janitor /janitor.py '
'-r {role} -p {principal} -z {zk} --auth_token={auth}')
shakedown.run_command_on_master(
janitor_cmd.format(
role=role,
principal=principal,
zk=zk,
auth=shakedown.run_dcos_command(
'config show core.dcos_acs_token')[0].strip()))
finish = time.time()
sdk_utils.out(
'Uninstall done after pkg({}) + janitor({}) = total({})'.format(
shakedown.pretty_duration(janitor_start - start),
shakedown.pretty_duration(finish - janitor_start),
shakedown.pretty_duration(finish - start)))
else:
sdk_utils.out('Uninstalling {}'.format(service_name))
try:
shakedown.uninstall_package_and_wait(package_name,
service_name=service_name)
# service_name may already contain a leading slash:
marathon_app_id = '/' + service_name.lstrip('/')
sdk_utils.out(
'Waiting for no deployments for {}'.format(marathon_app_id))
shakedown.deployment_wait(600, marathon_app_id)
# wait for service to be gone according to marathon
def marathon_dropped_service():
client = shakedown.marathon.create_client()
app_list = client.get_apps()
app_ids = [app['id'] for app in app_list]
sdk_utils.out('Marathon apps: {}'.format(app_ids))
matching_app_ids = [
app_id for app_id in app_ids if app_id == marathon_app_id
]
if len(matching_app_ids) > 1:
sdk_utils.out('Found multiple apps with id {}'.format(
marathon_app_id))
return len(matching_app_ids) == 0
sdk_utils.out(
'Waiting for no {} Marathon app'.format(marathon_app_id))
shakedown.time_wait(marathon_dropped_service)
except (dcos.errors.DCOSException, ValueError) as e:
sdk_utils.out(
'Got exception when uninstalling package: {}'.format(e))
finally:
sdk_utils.list_reserved_resources()
def get_elasticsearch_indices_stats(index_name, service_name=PACKAGE_NAME):
exit_status, output = shakedown.run_command_on_master(
"{}/{}/_stats'".format(_curl_api(service_name, "GET"), index_name))
return output
def get_kibana_status():
token = shakedown.authenticate('bootstrapuser', 'deleteme')
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}/kibana/login".format(
token, shakedown.dcos_service_url(PACKAGE_NAME))
exit_status, output = shakedown.run_command_on_master(curl_cmd)
return output
def create_index(index_name, params, service_name=PACKAGE_NAME):
exit_status, output = shakedown.run_command_on_master(
"{}/{}' -d '{}'".format(_curl_api(service_name, "PUT"), index_name,
json.dumps(params)))
return output
def http_output_check(stop_max_attempt_number=30):
status, output = shakedown.run_command_on_master('curl {}'.format(relay_url))
assert status
assert 'Pong /pinger' in output
assert 'Relay from /relay' in output
def get_xpack_license():
command = "{}/_xpack/license'".format(curl_api("GET"))
exit_status, output = shakedown.run_command_on_master(command)
return output
def check_dns():
cmd = 'ping -c 1 {}.{}.mesos'.format(app_id, marathon_service_name)
shakedown.wait_for_dns('{}.{}.mesos'.format(app_id, marathon_service_name))
status, output = shakedown.run_command_on_master(cmd)
assert status
def _get_elasticsearch_index_health(curl_api, index_name):
exit_status, output = shakedown.run_command_on_master(
"{}/_cluster/health/{}'".format(curl_api, index_name))
return output
def restart_master_node():
"""Restarts the master node."""
shakedown.run_command_on_master("sudo /sbin/shutdown -r now")
def _get_elasticsearch_cluster_health(curl_api):
exit_status, output = shakedown.run_command_on_master(
"{}/_cluster/health'".format(curl_api))
return output
def value_check():
status, data = shakedown.run_command_on_master(cmd)
assert status, "{} did not succeed".format(cmd)
assert data.rstrip() == secret_value
def check_task(cmd, target_data):
run, data = shakedown.run_command_on_master(cmd)
assert run, "{} did not succeed".format(cmd)
assert data == target_data, "'{}' was not equal to {}".format(
data, target_data)
