def _run_janitor(service_name):
janitor_cmd = (
'docker run mesosphere/janitor /janitor.py '
'-r {svc}-role -p {svc}-principal -z dcos-service-{svc} --auth_token={auth}')
shakedown.run_command_on_master(janitor_cmd.format(
svc=service_name,
auth=shakedown.dcos_acs_token()))
def _copy_file_to_localhost(self):
"""
Copies the keytab that was generated inside the container running the KDC server to the localhost
so it can be uploaded to the secret store later.
The keytab will end up in path: <temp_working_dir>/<keytab_file>
"""
log.info("Copying {} to the temp working directory".format(
self.keytab_file_name))
keytab_absolute_path = "{mesos_agents_path}/{host_id}/frameworks/{framework_id}/executors/{task_id}/runs/latest/{keytab_file}".format(
mesos_agents_path="/var/lib/mesos/slave/slaves",
host_id=self.kdc_host_id,
framework_id=self.framework_id,
task_id=self.task_id,
keytab_file=self.keytab_file_name)
keytab_url = "{cluster_url}/slave/{agent_id}/files/download?path={path}".format(
cluster_url=shakedown.dcos_url(),
agent_id=self.kdc_host_id,
path=keytab_absolute_path)
dest = "{temp_working_dir}/{keytab_file}".format(
temp_working_dir=self.temp_working_dir.name,
keytab_file=self.keytab_file_name)
curl_cmd = "curl -k --header '{auth}' {url} > {dest_file}".format(
auth="Authorization: token={token}".format(
token=shakedown.dcos_acs_token()),
url=keytab_url,
dest_file=dest)
try:
run([curl_cmd], shell=True)
except Exception as e:
raise RuntimeError("Failed to download the keytab file: {}".format(
repr(e)))
def events_to_file():
leader_ip = shakedown.marathon_leader_ip()
print("entering events_to_file fixture")
shakedown.run_command(leader_ip, 'rm events.txt')
# In strict mode marathon runs in SSL mode on port 8443 and requires authentication
if shakedown.ee_version() == 'strict':
shakedown.run_command(
leader_ip,
'(curl --compressed -H "Cache-Control: no-cache" -H "Accept: text/event-stream" '
+ '-H "Authorization: token={}" '.format(
shakedown.dcos_acs_token()) +
'-o events.txt -k https://marathon.mesos:8443/v2/events; echo $? > events.exitcode) &'
)
# Otherwise marathon runs on HTTP mode on port 8080
else:
shakedown.run_command(
leader_ip,
'(curl --compressed -H "Cache-Control: no-cache" -H "Accept: text/event-stream" '
'-o events.txt http://marathon.mesos:8080/v2/events; echo $? > events.exitcode) &'
)
yield
shakedown.kill_process_on_host(leader_ip, '[c]url')
shakedown.run_command(leader_ip, 'rm events.txt')
shakedown.run_command(leader_ip, 'rm events.exitcode')
print("exiting events_to_file fixture")
def _run_janitor():
janitor_cmd = (
'docker run mesosphere/janitor /janitor.py '
'-r spark-role -p spark-principal -z spark_mesos_dispatcher --auth_token={auth}'
)
shakedown.run_command_on_master(
janitor_cmd.format(auth=shakedown.dcos_acs_token()))
def master_service_status_code(url):
auth = DCOSAcsAuth(shakedown.dcos_acs_token())
response = requests.get(url=url,
timeout=5,
auth=auth,
verify=verify_ssl())
return response.status_code
def test_jar():
master_url = ("https" if _is_strict() else "http") + "://leader.mesos:5050"
spark_job_runner_args = '{} dcos \\"*\\" spark:only 2 --auth-token={}'.format(
master_url, shakedown.dcos_acs_token())
jar_url = _upload_file(os.getenv('TEST_JAR_PATH'))
_run_tests(jar_url, spark_job_runner_args, "All tests passed", [
"--class",
'com.typesafe.spark.test.mesos.framework.runners.SparkJobRunner'
])
def test_jar(service_name=utils.SPARK_SERVICE_NAME):
master_url = ("https" if sdk_utils.is_strict_mode() else "http") + "://leader.mesos:5050"
spark_job_runner_args = '{} dcos \\"*\\" spark:only 2 --auth-token={}'.format(
master_url,
shakedown.dcos_acs_token())
utils.run_tests(app_url=utils.upload_mesos_test_jar(),
app_args=spark_job_runner_args,
expected_output="All tests passed",
service_name=service_name,
args=['--class com.typesafe.spark.test.mesos.framework.runners.SparkJobRunner'])
def check_kibana_adminrouter_integration(path):
dcos_token = shakedown.dcos_acs_token()
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}/{}".format(
dcos_token, shakedown.dcos_url().rstrip('/'), path.lstrip('/'))
def fun():
exit_status, output = shakedown.run_command_on_master(curl_cmd)
return output and "HTTP/1.1 200" in output
return shakedown.wait_for(fun, timeout_seconds=DEFAULT_KIBANA_TIMEOUT, noisy=True)
def test_jar(app_name="/spark"):
master_url = ("https" if utils.is_strict() else "http") + "://leader.mesos:5050"
spark_job_runner_args = '{} dcos \\"*\\" spark:only 2 --auth-token={}'.format(
master_url,
shakedown.dcos_acs_token())
jar_url = utils.upload_file(os.getenv('TEST_JAR_PATH'))
utils.run_tests(app_url=jar_url,
app_args=spark_job_runner_args,
expected_output="All tests passed",
app_name=app_name,
args=["--class", 'com.typesafe.spark.test.mesos.framework.runners.SparkJobRunner'])
async def sse_events():
url = urljoin(shakedown.dcos_url(), 'service/marathon/v2/events')
headers = {
'Authorization': 'token={}'.format(shakedown.dcos_acs_token()),
'Accept': 'text/event-stream'
}
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url) as response:
async def internal_generator():
client = SSEClient(response.content)
async for event in client.events():
yield json.loads(event.data)
yield internal_generator()
async def sse_events():
url = urljoin(shakedown.dcos_url(), 'service/marathon/v2/events')
headers = {'Authorization': 'token={}'.format(shakedown.dcos_acs_token()),
'Accept': 'text/event-stream'}
ssl_context = get_ssl_context()
verify_ssl = ssl_context is not None
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url, verify_ssl=verify_ssl, ssl_context=ssl_context) as response:
async def internal_generator():
client = SSEClient(response.content)
async for event in client.events():
yield json.loads(event.data)
yield internal_generator()
def _block_on_adminrouter(master_ip):
headers = {'Authorization': "token={}".format(shakedown.dcos_acs_token())}
metadata_url = "http://{}/metadata".format(master_ip)
def get_metadata():
response = requests.get(metadata_url, headers=headers)
return response
def success(response):
error_message = "Failed to parse json"
try:
is_healthy = response.json()['PUBLIC_IPV4'] == master_ip
return is_healthy, "Master is not healthy yet"
except Exception:
return False, error_message
spin(get_metadata, success, HEALTH_WAIT_TIME)
log.info("Master is up again. Master IP: {}".format(master_ip))
def check_service_availability_on_master(master_ip, service):
schema = 'https' if ee_version() == 'strict' or ee_version(
) == 'permissive' else 'http'
url = "{}://{}/service/{}/{}".format(schema, master_ip, service, path)
auth = DCOSAcsAuth(shakedown.dcos_acs_token())
try:
response = requests.get(url=url,
timeout=5,
auth=auth,
verify=verify_ssl())
except ReadTimeout as e:
raise DCOSException("service " + service_name +
" is unavailable at " + master_ip)
if response.status_code == 200:
return True
else:
print(response)
raise DCOSException("service " + service_name +
" is unavailable at " + master_ip)
def check_kibana_adminrouter_integration(path):
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}/{}".format(
shakedown.dcos_acs_token(), shakedown.dcos_url().rstrip('/'), path.lstrip('/'))
exit_ok, output = sdk_cmd.master_ssh(curl_cmd)
return exit_ok and output and "HTTP/1.1 200" in output
def check_kibana_adminrouter_integration(path):
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}/{}".format(
shakedown.dcos_acs_token(),
shakedown.dcos_url().rstrip('/'), path.lstrip('/'))
exit_ok, output = shakedown.run_command_on_master(curl_cmd)
return exit_ok and output and "HTTP/1.1 200" in output
import os
import retrying
import shakedown
import time
from common import (app, block_port, cluster_info, delete_all_apps_wait, ensure_mom,
ip_of_mom, ip_other_than_mom, pin_to_host, systemctl_master)
from dcos import marathon
from shakedown import dcos_1_8, dcos_version_less_than, private_agent_2, required_private_agents
from utils import fixture_dir, get_resource, marathon_on_marathon
PACKAGE_NAME = 'marathon'
PACKAGE_APP_ID = 'marathon-user'
DCOS_SERVICE_URL = shakedown.dcos_service_url(PACKAGE_APP_ID)
TOKEN = shakedown.dcos_acs_token()
def setup_module(module):
# verify test system requirements are met (number of nodes needed)
ensure_mom()
shakedown.wait_for_service_endpoint(PACKAGE_APP_ID)
cluster_info()
def setup_function(function):
shakedown.wait_for_service_endpoint('marathon-user')
with marathon_on_marathon():
delete_all_apps_wait()
