def ensure_permissions():
common.set_service_account_permissions(MOM_EE_SERVICE_ACCOUNT)
url = urljoin(shakedown.dcos_url(), 'acs/api/v1/acls/dcos:superuser/users/{}'.format(MOM_EE_SERVICE_ACCOUNT))
req = http.get(url)
expected = '/acs/api/v1/acls/dcos:superuser/users/{}/full'.format(MOM_EE_SERVICE_ACCOUNT)
assert req.json()['array'][0]['url'] == expected, "Service account permissions couldn't be set"
def set_service_account_permissions(service_account,
resource='dcos:superuser',
action='full'):
"""Set permissions for given `{service_account}` for passed `{resource}` with
`{action}`. For more information consult the DC/OS documentation:
https://docs.mesosphere.com/1.9/administration/id-and-access-mgt/permissions/user-service-perms/
"""
try:
print('Granting {} permissions to {}/users/{}'.format(
action, resource, service_account))
url = urljoin(
shakedown.dcos_url(),
'acs/api/v1/acls/{}/users/{}/{}'.format(resource, service_account,
action))
req = http.put(url)
msg = 'Failed to grant permissions to the service account: {}, {}'.format(
req, req.text)
assert req.status_code == 204, msg
except DCOSHTTPException as e:
if (e.response.status_code == 409):
print('Service account {} already has {} permissions set'.format(
service_account, resource))
else:
print("Unexpected HTTP error: {}".format(e.response))
raise
except:
print("Unexpected error:", sys.exc_info()[0])
raise
def add_acs_resource(resource):
"""Create given ACS `{resource}`. For more information consult the DC/OS documentation:
https://docs.mesosphere.com/1.9/administration/id-and-access-mgt/permissions/user-service-perms/
"""
import json
try:
print('Adding ACS resource: {}'.format(resource))
url = urljoin(shakedown.dcos_url(),
'acs/api/v1/acls/{}'.format(resource))
extra_args = {'headers': {'Content-Type': 'application/json'}}
req = http.put(url,
data=json.dumps({'description': resource}),
**extra_args)
assert req.status_code == 201, 'Failed create ACS resource: {}, {}'.format(
req, req.text)
except DCOSHTTPException as e:
if (e.response.status_code == 409):
print('ACS resource {} already exists'.format(resource))
else:
print("Unexpected HTTP error: {}, {}".format(
e.response, e.response.text))
raise
except:
print("Unexpected error:", sys.exc_info()[0])
raise
def _copy_file_to_localhost(self):
"""
Copies the keytab that was generated inside the container running the KDC server to the localhost
so it can be uploaded to the secret store later.
The keytab will end up in path: <temp_working_dir>/<keytab_file>
"""
log.info("Copying {} to the temp working directory".format(
self.keytab_file_name))
keytab_absolute_path = "{mesos_agents_path}/{host_id}/frameworks/{framework_id}/executors/{task_id}/runs/latest/{keytab_file}".format(
mesos_agents_path="/var/lib/mesos/slave/slaves",
host_id=self.kdc_host_id,
framework_id=self.framework_id,
task_id=self.task_id,
keytab_file=self.keytab_file_name)
keytab_url = "{cluster_url}/slave/{agent_id}/files/download?path={path}".format(
cluster_url=shakedown.dcos_url(),
agent_id=self.kdc_host_id,
path=keytab_absolute_path)
dest = "{temp_working_dir}/{keytab_file}".format(
temp_working_dir=self.temp_working_dir.name,
keytab_file=self.keytab_file_name)
curl_cmd = "curl -k --header '{auth}' {url} > {dest_file}".format(
auth="Authorization: token={token}".format(
token=shakedown.dcos_acs_token()),
url=keytab_url,
dest_file=dest)
try:
run([curl_cmd], shell=True)
except Exception as e:
raise RuntimeError("Failed to download the keytab file: {}".format(
repr(e)))
def check_kibana_adminrouter_integration(path):
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}/{}".format(
DCOS_TOKEN, shakedown.dcos_url().rstrip('/'), path.lstrip('/'))
def fun():
exit_status, output = shakedown.run_command_on_master(curl_cmd)
return output and "HTTP/1.1 200" in output
return shakedown.wait_for(fun, timeout_seconds=DEFAULT_KIBANA_TIMEOUT, noisy=True)
def set_service_account_permissions(service_account, ressource='dcos:superuser', action='full'):
"""Set permissions for given `{service_account}` for passed `{ressource}` with
`{action}`. For more information consult the DC/OS documentation:
https://docs.mesosphere.com/1.9/administration/id-and-access-mgt/permissions/user-service-perms/
"""
print('Granting {} permissions to {}/users/{}'.format(action, ressource, service_account))
url = urljoin(shakedown.dcos_url(), 'acs/api/v1/acls/{}/users/{}/{}'.format(ressource, service_account, action))
req = http.put(url)
assert req.status_code == 204, 'Failed to grant permissions to the service account: {}, {}'.format(req, req.text)
def test_non_authorized_user():
with shakedown.new_dcos_user('kenny', 'kenny'):
with pytest.raises(
dcos.errors.DCOSAuthorizationException) as exec_info:
_ = dcos.http.get(
urljoin(shakedown.dcos_url(), 'service/marathon/v2/apps'))
error = exec_info.value
assert str(
error) == "You are not authorized to perform this operation"
def test_non_authenticated_user():
with shakedown.no_user():
with pytest.raises(
dcos.errors.DCOSAuthenticationException) as exec_info:
_ = dcos.http.get(
urljoin(shakedown.dcos_url(), 'service/marathon/v2/apps'))
error = exec_info.value
assert str(
error) == "Authentication failed. Please run `dcos auth login`"
def get_metrics(package_name, service_name, task_name):
"""Return a list of metrics datapoints.
Keyword arguments:
service_name -- the name of the service to get metrics for
task_name -- the name of the task whose agent to run metrics commands from
"""
tasks = shakedown.get_service_tasks(service_name)
for task in tasks:
if task['name'] == task_name:
task_to_check = task
if task_to_check is None:
raise Exception("Could not find task")
agent_id = task_to_check['slave_id']
executor_id = task_to_check['executor_id']
# TODO: uncomment the following block of comments when the /containers endpoint reports the correct container IDs
# and remove the code following the comments that gets the correct container ID via 'pod info'
## Fetch the list of containers for the agent
#containers_url = "{}/system/v1/agent/{}/metrics/v0/containers".format(shakedown.dcos_url(), agent_id)
#containers_response = sdk_cmd.request("GET", containers_url, retry=False)
#if containers_response.ok is None:
# log.info("Unable to fetch containers list")
# raise Exception("Unable to fetch containers list: {}".format(containers_url))
# instead of receiving the pod name in this function's parameter list, extract
# the name of the pod from the task name to not break the code when the
# above comment-block is uncommented
pod_name = '-'.join(task_name.split("-")[:2])
pod_info = sdk_cmd.svc_cli(package_name, service_name, "pod info {}".format(pod_name), json=True)
task_info = None
for task in pod_info:
if task["info"]["name"] == task_name:
task_info = task
break
if not task_info:
return []
container_id = task_info["status"]["containerStatus"]["containerId"]["value"]
#for container_id in json.loads(containers_response.text):
app_url = "{}/system/v1/agent/{}/metrics/v0/containers/{}/app".format(
shakedown.dcos_url(), agent_id, container_id)
app_response = sdk_cmd.request("GET", app_url, retry=False)
if app_response.ok is None:
raise("Failed to get metrics from container")
#continue
app_json = json.loads(app_response.text)
if app_json['dimensions']['executor_id'] == executor_id:
return app_json['datapoints']
raise Exception("No metrics found")
def check_kibana_adminrouter_integration(path):
dcos_token = shakedown.dcos_acs_token()
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}/{}".format(
dcos_token, shakedown.dcos_url().rstrip('/'), path.lstrip('/'))
def fun():
exit_status, output = shakedown.run_command_on_master(curl_cmd)
return output and "HTTP/1.1 200" in output
return shakedown.wait_for(fun, timeout_seconds=DEFAULT_KIBANA_TIMEOUT, noisy=True)
def check_kibana_adminrouter_integration(path):
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}/{}".format(
DCOS_TOKEN,
shakedown.dcos_url().rstrip('/'), path.lstrip('/'))
def fun():
exit_status, output = shakedown.run_command_on_master(curl_cmd)
return output and "HTTP/1.1 200" in output
return shakedown.wait_for(fun,
timeout_seconds=KIBANA_WAIT_TIME_IN_SECONDS,
noisy=True)
def get_metrics(service_name, task_name):
"""Return a list of metrics datapoints.
Keyword arguments:
service_name -- the name of the service to get metrics for
task_name -- the name of the task whose agent to run metrics commands from
"""
tasks = shakedown.get_service_tasks(service_name)
for task in tasks:
if task['name'] == task_name:
task_to_check = task
if task_to_check is None:
raise Exception("Could not find task")
agent_id = task_to_check['slave_id']
executor_id = task_to_check['executor_id']
# Fetch the list of containers for the agent
containers_url = "{}/system/v1/agent/{}/metrics/v0/containers".format(
shakedown.dcos_url(), agent_id)
containers_response = cmd.request("GET", containers_url, retry=False)
if containers_response.ok is None:
log.info("Unable to fetch containers list")
raise Exception(
"Unable to fetch containers list: {}".format(containers_url))
for container in json.loads(containers_response.text):
app_url = "{}/system/v1/agent/{}/metrics/v0/containers/{}/app".format(
shakedown.dcos_url(), agent_id, container)
app_response = cmd.request("GET", app_url, retry=False)
if app_response.ok is None:
continue
app_json = json.loads(app_response.text)
if app_json['dimensions']['executor_id'] == executor_id:
return app_json['datapoints']
raise Exception("No metrics found")
async def sse_events():
url = urljoin(shakedown.dcos_url(), 'service/marathon/v2/events')
headers = {'Authorization': 'token={}'.format(shakedown.dcos_acs_token()),
'Accept': 'text/event-stream'}
ssl_context = get_ssl_context()
verify_ssl = ssl_context is not None
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url, verify_ssl=verify_ssl, ssl_context=ssl_context) as response:
async def internal_generator():
client = SSEClient(response.content)
async for event in client.events():
yield json.loads(event.data)
yield internal_generator()
async def sse_events():
url = urljoin(shakedown.dcos_url(), 'service/marathon/v2/events')
headers = {
'Authorization': 'token={}'.format(shakedown.dcos_acs_token()),
'Accept': 'text/event-stream'
}
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url) as response:
async def internal_generator():
client = SSEClient(response.content)
async for event in client.events():
yield json.loads(event.data)
yield internal_generator()
def set_service_account_permissions(service_account, resource='dcos:superuser', action='full'):
"""Set permissions for given `{service_account}` for passed `{resource}` with
`{action}`. For more information consult the DC/OS documentation:
https://docs.mesosphere.com/1.9/administration/id-and-access-mgt/permissions/user-service-perms/
"""
try:
print('Granting {} permissions to {}/users/{}'.format(action, resource, service_account))
url = urljoin(shakedown.dcos_url(), 'acs/api/v1/acls/{}/users/{}/{}'.format(resource, service_account, action))
req = http.put(url)
msg = 'Failed to grant permissions to the service account: {}, {}'.format(req, req.text)
assert req.status_code == 204, msg
except DCOSHTTPException as e:
if (e.response.status_code == 409):
print('Service account {} already has {} permissions set'.format(service_account, resource))
else:
print("Unexpected HTTP error: {}".format(e.response))
raise
except Exception:
print("Unexpected error:", sys.exc_info()[0])
raise
def add_acs_resource(resource):
"""Create given ACS `{resource}`. For more information consult the DC/OS documentation:
https://docs.mesosphere.com/1.9/administration/id-and-access-mgt/permissions/user-service-perms/
"""
import json
try:
print('Adding ACS resource: {}'.format(resource))
url = urljoin(shakedown.dcos_url(), 'acs/api/v1/acls/{}'.format(resource))
extra_args = {'headers': {'Content-Type': 'application/json'}}
req = http.put(url, data=json.dumps({'description': resource}), **extra_args)
assert req.status_code == 201, 'Failed create ACS resource: {}, {}'.format(req, req.text)
except DCOSHTTPException as e:
if (e.response.status_code == 409):
print('ACS resource {} already exists'.format(resource))
else:
print("Unexpected HTTP error: {}, {}".format(e.response, e.response.text))
raise
except Exception:
print("Unexpected error:", sys.exc_info()[0])
raise
def get_marathon_uri():
"""Gets URL to the Marathon instance"""
return '{}/marathon'.format(shakedown.dcos_url())
def test_non_authenicated_user():
with shakedown.no_user():
with pytest.raises(dcos.errors.DCOSAuthenticationException) as exec_info:
response = dcos.http.get(urljoin(shakedown.dcos_url(), 'service/marathon/v2/apps'))
error = exc_info.value
assert str(error) == "Authentication failed. Please run `dcos auth login`"
def get_marathon_uri():
"""Gets URL to the Marathon instance"""
return '{}/marathon'.format(shakedown.dcos_url())
def _acl_url():
return urljoin(dcos_url(), 'acs/api/v1/')
def check_kibana_adminrouter_integration(path):
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}/{}".format(
shakedown.dcos_acs_token(), shakedown.dcos_url().rstrip('/'), path.lstrip('/'))
exit_ok, output = sdk_cmd.master_ssh(curl_cmd)
return exit_ok and output and "HTTP/1.1 200" in output
def test_non_authorized_user():
with shakedown.new_dcos_user('kenny', 'kenny'):
with pytest.raises(dcos.errors.DCOSAuthorizationException) as exec_info:
response = dcos.http.get(urljoin(shakedown.dcos_url(), 'service/marathon/v2/apps'))
error = exc_info.value
assert str(error) == "You are not authorized to perform this operation"
def unit_health_url(unit_and_node=''):
return '{}/system/health/v1/units/{}'.format(shakedown.dcos_url(),
unit_and_node)
def get_metrics(package_name, service_name, task_name):
"""Return a list of metrics datapoints.
Keyword arguments:
package_name -- the name of the package the service is using
service_name -- the name of the service to get metrics for
task_name -- the name of the task whose agent to run metrics commands from
"""
tasks = shakedown.get_service_tasks(service_name)
for task in tasks:
if task['name'] == task_name:
task_to_check = task
if task_to_check is None:
raise Exception("Could not find task")
agent_id = task_to_check['slave_id']
executor_id = task_to_check['executor_id']
pod_name = '-'.join(task_name.split("-")[:2])
pod_info = sdk_cmd.svc_cli(package_name,
service_name,
"pod info {}".format(pod_name),
json=True)
task_info = None
for task in pod_info:
if task["info"]["name"] == task_name:
task_info = task
break
if not task_info:
return []
task_container_id = task_info["status"]["containerStatus"]["containerId"][
"value"]
# Not related to functionality but consuming this
# endpoint to verify downstream integrity
containers_url = "{}/system/v1/agent/{}/metrics/v0/containers".format(
shakedown.dcos_url(), agent_id)
containers_response = sdk_cmd.request("GET", containers_url, retry=False)
if containers_response.ok is None:
log.info("Unable to fetch containers list")
raise Exception(
"Unable to fetch containers list: {}".format(containers_url))
reported_container_ids = json.loads(containers_response.text)
container_id_reported = False
for container_id in reported_container_ids:
if container_id == task_container_id:
container_id_reported = True
if not container_id_reported:
raise ValueError(
"The metrics /container endpoint returned {}, expecting {} to be returned as well"
.format(reported_container_ids, task_container_id))
app_url = "{}/system/v1/agent/{}/metrics/v0/containers/{}/app".format(
shakedown.dcos_url(), agent_id, task_container_id)
app_response = sdk_cmd.request("GET", app_url, retry=False)
if app_response.ok is None:
raise ValueError("Failed to get metrics from container")
app_json = json.loads(app_response.text)
if app_json['dimensions']['executor_id'] == executor_id:
return app_json['datapoints']
raise Exception("No metrics found")
def get_kibana_status(path):
token = shakedown.authenticate('bootstrapuser', 'deleteme')
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}{}".format(
token, shakedown.dcos_url(), path)
exit_status, output = shakedown.run_command_on_master(curl_cmd)
return output
def check_cluster_exists():
response = http.get(shakedown.dcos_url())
assert response.status_code == 200
def check_kibana_adminrouter_integration(path):
curl_cmd = "curl -I -k -H \"Authorization: token={}\" -s {}/{}".format(
shakedown.dcos_acs_token(),
shakedown.dcos_url().rstrip('/'), path.lstrip('/'))
exit_ok, output = shakedown.run_command_on_master(curl_cmd)
return exit_ok and output and "HTTP/1.1 200" in output
def check_cluster_exists():
response = http.get(shakedown.dcos_url())
assert response.status_code == 200