def zookeeper_server(configure_security):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"virtual_network_enabled": True,
}
}
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
try:
sdk_install.uninstall(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
)
if sdk_utils.is_strict_mode():
service_options = sdk_utils.merge_dictionaries(
{
"service": {
"service_account": zk_account,
"service_account_secret": zk_secret,
}
},
service_options,
)
sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
service_account=zk_account,
service_account_secret=zk_secret,
)
sdk_install.install(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
package_version=config.ZOOKEEPER_PACKAGE_VERSION,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False,
)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
},
}
finally:
sdk_install.uninstall(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
)
if sdk_utils.is_strict_mode():
sdk_security.delete_service_account(
service_account_name=zk_account,
service_account_secret=zk_secret,
)
def _submit_consumer(name,
spark_executor_docker_image,
jar,
kafka_broker_dns,
cassandra_native_client_dns,
dispatcher,
kafka_topics,
kafka_group_id,
write_to_cassandra,
batch_size_seconds,
cassandra_keyspace,
cassandra_table,
spark_cores_max,
spark_executor_cores,
must_fail: bool):
app_args = ["--appName", name,
"--brokers", ",".join(kafka_broker_dns),
"--topics", kafka_topics,
"--groupId", kafka_group_id,
"--batchSizeSeconds", str(batch_size_seconds),
"--cassandraKeyspace", cassandra_keyspace,
"--cassandraTable", cassandra_table]
if must_fail:
app_args.extend(["--mustFailDueToInvalidArgument"])
if not write_to_cassandra:
app_args.extend(["--shouldNotWriteToCassandra"])
cassandra_hosts = map(lambda x: x.split(':')[0], cassandra_native_client_dns)
cassandra_port = cassandra_native_client_dns[0].split(':')[1]
app_config = ["--supervise",
"--conf", "spark.cores.max={}".format(spark_cores_max),
"--conf", "spark.executor.cores={}".format(spark_executor_cores),
"--conf", "spark.cassandra.connection.host={}".format(",".join(cassandra_hosts)),
"--conf", "spark.cassandra.connection.port={}".format(cassandra_port),
"--name", name,
"--class", CONSUMER_CLASS_NAME]
if spark_executor_docker_image:
app_config.extend([
"--conf", "spark.mesos.executor.docker.image={}".format(spark_executor_docker_image)
])
args = app_config + COMMON_CONF
submission_id = spark_utils.submit_job(
app_url=jar,
app_args=" ".join(str(a) for a in app_args),
args=args,
verbose=False,
service_name=dispatcher['service']['name'],
driver_role=dispatcher['roles']['executors'],
spark_user=dispatcher['service']['user'] if sdk_utils.is_strict_mode() else None,
principal=dispatcher['service']['service_account'] if sdk_utils.is_strict_mode() else None)
return submission_id
def configure_role_permissions(configure_security_spark):
try:
if sdk_utils.is_strict_mode():
utils.grant_user_permissions("nobody", driver_role, utils.SPARK_SERVICE_ACCOUNT)
utils.grant_launch_task_permission(service_name)
yield
finally:
if sdk_utils.is_strict_mode():
utils.revoke_user_permissions("nobody", driver_role, utils.SPARK_SERVICE_ACCOUNT)
utils.revoke_launch_task_permission(service_name)
def submit_job(app_url: str, app_args: str, dispatcher: typing.Dict, duration: int, config: typing.List[str]):
dispatcher_name = dispatcher["service"]["name"]
log.info("Submitting job to dispatcher: %s, with duration: %s min.", dispatcher_name, duration)
spark_utils.submit_job(
service_name=dispatcher_name,
app_url=app_url,
app_args=app_args,
verbose=False,
args=config,
driver_role=dispatcher["roles"]["executors"],
spark_user=dispatcher["service"]["user"] if sdk_utils.is_strict_mode() else None,
principal=dispatcher["service"]["service_account"] if sdk_utils.is_strict_mode() else None)
def spark_security_session():
'''
Spark strict mode setup is slightly different from dcos-commons, so can't use sdk_security::security_session.
Differences:
(1) the role is "*", (2) the driver itself is a framework and needs permission to execute tasks.
'''
user = SPARK_USER
role = '*'
service_account = SPARK_SERVICE_ACCOUNT
secret = SPARK_SERVICE_ACCOUNT_SECRET
def grant_driver_permission(service_account_name, service_name):
app_id = "/{}".format(service_name.lstrip("/"))
# double-encoded (why?)
app_id = urllib.parse.quote(urllib.parse.quote(app_id, safe=''),
safe='')
sdk_security._grant(
service_account_name,
"dcos:mesos:master:task:app_id:{}".format(app_id),
description="Spark drivers may execute Mesos tasks",
action="create")
def setup_security():
LOGGER.info('Setting up strict-mode security for Spark')
sdk_security.create_service_account(
service_account_name=service_account,
service_account_secret=secret)
sdk_security.grant_permissions(linux_user=user,
role_name=role,
service_account_name=service_account)
grant_driver_permission(service_account, SPARK_SERVICE_NAME)
grant_driver_permission(service_account, FOLDERED_SPARK_SERVICE_NAME)
LOGGER.info('Finished setting up strict-mode security for Spark')
def cleanup_security():
LOGGER.info('Cleaning up strict-mode security for Spark')
sdk_security.revoke_permissions(linux_user=user,
role_name=role,
service_account_name=service_account)
sdk_security.delete_service_account(service_account, secret)
LOGGER.info('Finished cleaning up strict-mode security for Spark')
try:
if sdk_utils.is_strict_mode():
setup_security()
yield
finally:
if sdk_utils.is_strict_mode():
cleanup_security()
def configure_user_permissions(configure_security_spark):
"""Tests in this module require permissions for 'root' user, we can't use `spark_utils.spark_security_session` here
because it drops service account completely during teardown. To lower down amount of boilerplate code in tests
'configure_security_spark' fixture is defined with 'session' scope thus dropped service account won't be
recreated in tests executed after the current one.
"""
try:
if sdk_utils.is_strict_mode():
utils.grant_launch_task_permission(SERVICE_NAME)
utils.grant_user_permissions("root")
yield
finally:
if sdk_utils.is_strict_mode():
utils.revoke_launch_task_permission(SERVICE_NAME)
utils.revoke_user_permissions("root")
def _submit_producer(name,
spark_executor_docker_image,
jar,
kafka_broker_dns,
dispatcher,
kafka_topics,
number_of_words,
words_per_second,
spark_cores_max,
spark_executor_cores,
must_fail: bool):
app_args = ["--appName", name,
"--brokers", ",".join(kafka_broker_dns),
"--topics", kafka_topics,
"--numberOfWords", str(number_of_words),
"--wordsPerSecond", str(words_per_second)]
if must_fail:
app_args.extend(["--mustFailDueToInvalidArgument", ])
app_config = ["--conf", "spark.cores.max={}".format(spark_cores_max),
"--conf", "spark.executor.cores={}".format(spark_executor_cores),
"--name", name,
"--class", PRODUCER_CLASS_NAME]
# `number_of_words == 0` means infinite stream, so we'd like to have it
# restarted in the case of failures.
if number_of_words == 0:
app_config.extend(["--supervise"])
if spark_executor_docker_image:
app_config.extend([
"--conf", "spark.mesos.executor.docker.image={}".format(spark_executor_docker_image)
])
args = app_config + COMMON_CONF
submission_id = spark_utils.submit_job(
app_url=jar,
app_args=" ".join(str(a) for a in app_args),
args=args,
verbose=False,
service_name=dispatcher['service']['name'],
driver_role=dispatcher['roles']['executors'],
spark_user=dispatcher['service']['user'] if sdk_utils.is_strict_mode() else None,
principal=dispatcher['service']['service_account'] if sdk_utils.is_strict_mode() else None)
return submission_id
def submit_job(app_url,
app_args,
service_name=SPARK_SERVICE_NAME,
args=[],
spark_user=None,
driver_role=SPARK_DRIVER_ROLE,
verbose=True,
principal=SPARK_SERVICE_ACCOUNT,
use_cli=True):
conf_args = args.copy()
if driver_role:
conf_args += ['--conf', 'spark.mesos.role={}'.format(driver_role)]
if SPARK_DOCKER_USER is not None:
conf_args += [
'--conf', 'spark.mesos.executor.docker.parameters=user={}'.format(
SPARK_DOCKER_USER)
]
if not list(filter(lambda x: "spark.driver.memory=" in x, conf_args)):
conf_args += ['--conf', 'spark.driver.memory=2g']
if sdk_utils.is_strict_mode():
conf_args += ['--conf spark.mesos.principal={}'.format(principal)]
if spark_user is not None:
conf_args += [
'--conf spark.mesos.driverEnv.SPARK_USER={}'.format(spark_user)
]
if not list(filter(lambda x: "spark.mesos.containerizer=" in x,
conf_args)):
conf_args += ['--conf', 'spark.mesos.containerizer=mesos']
submit_args = ' '.join([' '.join(conf_args), app_url, app_args])
verbose_flag = "--verbose" if verbose else ""
result = None
if use_cli:
stdout = sdk_cmd.svc_cli(
SPARK_PACKAGE_NAME, service_name,
'run {} --submit-args="{}"'.format(verbose_flag, submit_args))
result = re.search(r"Submission id: (\S+)", stdout)
else:
docker_cmd = "sudo docker run --net=host -ti {} bin/spark-submit {}".format(
SPARK_DOCKER_IMAGE, submit_args)
ssh_opts = "--option UserKnownHostsFile=/dev/null --option StrictHostKeyChecking=no"
log.info("Running Docker command on leader: {}".format(docker_cmd))
_, stdout, stderr = sdk_cmd.run_raw_cli(
"node ssh --master-proxy --leader --user={} {} '{}'".format(
sdk_cmd.LINUX_USER, ssh_opts, docker_cmd))
result = re.search(r'"submissionId" : "(\S+)"', stdout)
if not result:
raise Exception(
"Unable to find submission ID in stdout:\n{}".format(stdout))
return result.group(1)
def security_session(
framework_name: str,
permissions: List[Dict[str, str]] = [],
linux_user: str = DEFAULT_LINUX_USER,
service_account: str = "service-acct",
service_account_secret: str = "secret",
) -> Iterator[None]:
"""Create a service account and configure permissions for strict-mode tests.
This should generally be used as a fixture in a framework's conftest.py:
@pytest.fixture(scope='session')
def configure_security(configure_universe):
yield from sdk_security.security_session(framework_name, permissions, linux_user, 'service-acct')
"""
is_strict = sdk_utils.is_strict_mode()
try:
if is_strict:
roles = _get_service_role(framework_name) + _get_integration_test_foldered_role(
framework_name
)
security_info = setup_security(
framework_name,
roles=roles,
permissions=permissions,
linux_user=linux_user,
service_account=service_account,
service_account_secret=service_account_secret,
)
yield
finally:
if is_strict:
cleanup_security(framework_name, security_info)
def kerberized_kafka(configure_security_kafka, kerberos_options):
try:
additional_options = {
"service": {
"name": KAFKA_SERVICE_NAME,
"security": kerberos_options
},
"kafka": {
"default_replication_factor": 3,
"num_partitions": 32
}
}
if sdk_utils.is_strict_mode():
additional_options["service"][
"service_account"] = KAFKA_SERVICE_ACCOUNT
additional_options["service"]["principal"] = KAFKA_SERVICE_ACCOUNT
additional_options["service"][
"service_account_secret"] = KAFKA_SERVICE_ACCOUNT_SECRET
additional_options["service"][
"secret_name"] = KAFKA_SERVICE_ACCOUNT_SECRET
sdk_install.uninstall(KAFKA_PACKAGE_NAME, KAFKA_SERVICE_NAME)
sdk_install.install(KAFKA_PACKAGE_NAME,
KAFKA_SERVICE_NAME,
DEFAULT_KAFKA_TASK_COUNT,
additional_options=additional_options,
timeout_seconds=30 * 60)
yield
finally:
sdk_install.uninstall(KAFKA_PACKAGE_NAME, KAFKA_SERVICE_NAME)
def configure_zookeeper(configure_security, install_zookeeper_stub):
service_options = {
"service": {
"virtual_network_enabled": True
}
}
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
if sdk_utils.is_strict_mode():
service_options = sdk_install.merge_dictionaries({
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
try:
sdk_install.uninstall(ZK_PACKAGE, ZK_SERVICE_NAME)
sdk_security.setup_security(ZK_SERVICE_NAME, zk_account, zk_secret)
sdk_install.install(
ZK_PACKAGE,
ZK_SERVICE_NAME,
6,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield
finally:
sdk_install.uninstall(ZK_PACKAGE, ZK_SERVICE_NAME)
def hdfs_with_kerberos(configure_security_hdfs, kerberos_options):
try:
additional_options = {
"service": {
"name": HDFS_SERVICE_NAME,
"security": kerberos_options
},
"hdfs": {
"security_auth_to_local":
hdfs_auth.get_principal_to_user_mapping()
}
}
if sdk_utils.is_strict_mode():
additional_options["service"][
"service_account"] = HDFS_SERVICE_ACCOUNT
additional_options["service"]["principal"] = HDFS_SERVICE_ACCOUNT
additional_options["service"][
"service_account_secret"] = HDFS_SERVICE_ACCOUNT_SECRET
additional_options["service"][
"secret_name"] = HDFS_SERVICE_ACCOUNT_SECRET
sdk_install.uninstall(HDFS_PACKAGE_NAME, HDFS_SERVICE_NAME)
sdk_install.install(HDFS_PACKAGE_NAME,
HDFS_SERVICE_NAME,
DEFAULT_HDFS_TASK_COUNT,
additional_options=additional_options,
timeout_seconds=30 * 60)
yield
finally:
sdk_install.uninstall(HDFS_PACKAGE_NAME, HDFS_SERVICE_NAME)
def configure_role_permissions(configure_security_spark):
try:
if sdk_utils.is_strict_mode():
for role in ROLES:
utils.grant_user_permissions("nobody", role,
utils.SPARK_SERVICE_ACCOUNT)
utils.grant_launch_task_permission("/dev/spark")
yield
finally:
if sdk_utils.is_strict_mode():
for role in ROLES:
utils.revoke_user_permissions("nobody", role,
utils.SPARK_SERVICE_ACCOUNT)
utils.revoke_launch_task_permission("/dev/spark")
def security_session(
framework_name: str,
permissions: List[Dict[str, str]] = [],
linux_user: str = DEFAULT_LINUX_USER,
service_account: str = "service-acct",
service_account_secret: str = "secret",
) -> Iterator[None]:
"""Create a service account and configure permissions for strict-mode tests.
This should generally be used as a fixture in a framework's conftest.py:
@pytest.fixture(scope='session')
def configure_security(configure_universe):
yield from sdk_security.security_session(framework_name, permissions, linux_user, 'service-acct')
"""
is_strict = sdk_utils.is_strict_mode()
try:
if is_strict:
roles = _get_service_role(
framework_name) + _get_integration_test_foldered_role(
framework_name)
security_info = setup_security(
framework_name,
roles=roles,
permissions=permissions,
linux_user=linux_user,
service_account=service_account,
service_account_secret=service_account_secret,
)
yield
finally:
if is_strict:
cleanup_security(framework_name, security_info)
def setup_security(framework_name: str,
service_account: str = "service-acct",
service_account_secret: str = "secret") -> dict:
create_service_account(service_account_name=service_account,
service_account_secret=service_account_secret)
service_account_info = {
"name": service_account,
"secret": service_account_secret
}
if not sdk_utils.is_strict_mode():
log.info("Skipping strict-mode security setup on non-strict cluster")
return service_account_info
log.info("Setting up strict-mode security")
grant_permissions(linux_user="******",
role_name="{}-role".format(framework_name),
service_account_name=service_account)
grant_permissions(
linux_user="******",
role_name="slave_public%252F{}-role".format(framework_name),
service_account_name=service_account)
grant_permissions(
linux_user="******",
role_name="test__integration__{}-role".format(framework_name),
service_account_name=service_account)
log.info("Finished setting up strict-mode security")
return service_account_info
def submit_job(app_url,
app_args,
service_name=SPARK_SERVICE_NAME,
args=[],
spark_user=SPARK_USER,
driver_role=SPARK_DRIVER_ROLE,
verbose=True,
principal=SPARK_SERVICE_ACCOUNT):
conf_args = args.copy()
conf_args += ['--conf', 'spark.mesos.role={}'.format(driver_role)]
if not list(
filter(lambda x: x.startswith("spark.driver.memory="), conf_args)):
conf_args += ['--conf', 'spark.driver.memory=2g']
if sdk_utils.is_strict_mode():
conf_args += [
'--conf spark.mesos.driverEnv.SPARK_USER={}'.format(spark_user),
'--conf spark.mesos.principal={}'.format(principal)
]
submit_args = ' '.join([' '.join(conf_args), app_url, app_args])
verbose_flag = "--verbose" if verbose else ""
stdout = sdk_cmd.svc_cli(
SPARK_PACKAGE_NAME, service_name,
'run {} --submit-args="{}"'.format(verbose_flag, submit_args))
result = re.search(r"Submission id: (\S+)", stdout)
if not result:
raise Exception(
"Unable to find submission ID in stdout:\n{}".format(stdout))
return result.group(1)
def setup_security(service_name: str,
linux_user: str="nobody",
service_account: str="service-acct",
service_account_secret: str="secret") -> dict:
create_service_account(service_account_name=service_account,
service_account_secret=service_account_secret)
service_account_info = {"name": service_account,
"secret": service_account_secret,
"linux_user": linux_user,
"roles": []
}
if not sdk_utils.is_strict_mode():
log.info("Skipping strict-mode security setup on non-strict cluster")
return service_account_info
log.info("Setting up strict-mode security")
service_account_info["roles"] = _get_role_list(service_name)
for role_name in service_account_info["roles"]:
grant_permissions(
linux_user=linux_user,
role_name=role_name,
service_account_name=service_account
)
log.info("Finished setting up strict-mode security")
return service_account_info
def spark_security_session(
users=[SPARK_USER],
service_names=[SPARK_SERVICE_NAME, FOLDERED_SPARK_SERVICE_NAME]):
'''
Spark strict mode setup is slightly different from dcos-commons, so can't use sdk_security::security_session.
Differences:
(1) the role is "*", (2) the driver itself is a framework and needs permission to execute tasks.
'''
role = '*'
service_account = SPARK_SERVICE_ACCOUNT
secret = SPARK_SERVICE_ACCOUNT_SECRET
def setup_security():
log.info('Setting up strict-mode security for Spark')
sdk_security.create_service_account(
service_account_name=service_account,
service_account_secret=secret)
for user in users:
grant_user_permissions(user, role, service_account)
for service_name in service_names:
grant_launch_task_permission(service_name)
log.info('Finished setting up strict-mode security for Spark')
def cleanup_security():
log.info('Cleaning up strict-mode security for Spark')
for user in users:
revoke_user_permissions(user, role, service_account)
# TODO: improve security setup/teardown to make it more fine-grained (allow different service names/accts/users)
# tracking issue: https://jira.mesosphere.com/browse/DCOS-50933
sdk_security.delete_service_account(service_account, secret)
log.info('Finished cleaning up strict-mode security for Spark')
try:
if not sdk_utils.is_open_dcos():
sdk_security.install_enterprise_cli()
if sdk_utils.is_strict_mode():
setup_security()
yield
finally:
if sdk_utils.is_strict_mode():
cleanup_security()
def install(
package_name: str,
service_name: str,
expected_running_tasks: int,
additional_options: dict = {},
package_version: PackageVersion = PackageVersion.STUB_UNIVERSE,
timeout_seconds: int = TIMEOUT_SECONDS,
wait_for_deployment: bool = True,
insert_strict_options: bool = True,
wait_for_all_conditions: bool = True,
) -> None:
start = time.time()
# If the package is already installed at this point, fail immediately.
if sdk_marathon.app_exists(service_name):
raise Exception(
"Service is already installed: {}".format(service_name))
if insert_strict_options and sdk_utils.is_strict_mode():
# strict mode requires correct principal and secret to perform install.
# see also: sdk_security.py
options = sdk_utils.merge_dictionaries(
{
"service": {
"service_account": "service-acct",
"principal": "service-acct",
"service_account_secret": "secret",
"secret_name": "secret",
}
},
additional_options,
)
else:
options = additional_options
# 1. Install package, wait for tasks, wait for marathon deployment
_retried_install_impl(
package_name, service_name,
expected_running_tasks, package_version.value if isinstance(
package_version, PackageVersion) else package_version, options,
timeout_seconds, wait_for_all_conditions)
# 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit)
# This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete
# state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app)
if wait_for_deployment:
# this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected
# total task count via FINISHED tasks, without actually completing deployment
log.info(
"Waiting for package={} service={} to finish deployment plan...".
format(package_name, service_name))
sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds)
log.info("Installed package={} service={} after {}".format(
package_name, service_name,
sdk_utils.pretty_duration(time.time() - start)))
global _installed_service_names
_installed_service_names.add(service_name)
def zookeeper_server(kerberos):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": kerberos.get_realm(),
"keytab_secret": kerberos.get_keytab_path(),
}
}
}
}
zk_account = "kafka-zookeeper-service-account"
zk_secret = "kakfa-zookeeper-secret"
if sdk_utils.is_strict_mode():
service_options = sdk_install.merge_dictionaries(
{
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
service_account_info = sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
linux_user="******",
service_account=zk_account,
service_account_secret=zk_secret)
sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
}
}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME,
service_account_info)
def test_jar(service_name=utils.SPARK_SERVICE_NAME):
master_url = ("https" if sdk_utils.is_strict_mode() else "http") + "://leader.mesos:5050"
spark_job_runner_args = '{} dcos \\"*\\" spark:only 2 --auth-token={}'.format(
master_url,
shakedown.dcos_acs_token())
utils.run_tests(app_url=utils.upload_mesos_test_jar(),
app_args=spark_job_runner_args,
expected_output="All tests passed",
service_name=service_name,
args=['--class com.typesafe.spark.test.mesos.framework.runners.SparkJobRunner'])
def install(
package_name,
service_name,
expected_running_tasks,
additional_options={},
package_version=None,
timeout_seconds=TIMEOUT_SECONDS,
wait_for_deployment=True,
insert_strict_options=True,
install_cli=True):
start = time.time()
# If the package is already installed at this point, fail immediately.
if sdk_marathon.app_exists(service_name):
raise dcos.errors.DCOSException('Service is already installed: {}'.format(service_name))
if insert_strict_options and sdk_utils.is_strict_mode():
# strict mode requires correct principal and secret to perform install.
# see also: sdk_security.py
options = merge_dictionaries({
'service': {
'service_account': 'service-acct',
'principal': 'service-acct',
'service_account_secret': 'secret',
'secret_name': 'secret'
}
}, additional_options)
else:
options = additional_options
# 1. Install package, wait for tasks, wait for marathon deployment
_retried_install_impl(
package_name,
service_name,
expected_running_tasks,
options,
package_version,
timeout_seconds,
install_cli)
# 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit)
# This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete
# state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app)
if wait_for_deployment:
# this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected
# total task count via FINISHED tasks, without actually completing deployment
log.info('Waiting for package={} service={} to finish deployment plan...'.format(
package_name, service_name))
sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds)
log.info('Installed package={} service={} after {}'.format(
package_name, service_name, shakedown.pretty_duration(time.time() - start)))
global _installed_service_names
_installed_service_names.add(service_name)
def setup_security(service_name: str, linux_user: str) -> typing.Dict:
"""
Adds a service account and secret for the specified service name.
"""
if not sdk_utils.is_strict_mode():
return {}
service_account = normalize_string("{}-service-account".format(service_name))
service_account_secret = "{}-service-account-secret".format(service_name)
return sdk_security.setup_security(service_name,
linux_user,
service_account, service_account_secret)
def install(
package_name,
service_name,
expected_running_tasks,
additional_options={},
package_version=None,
timeout_seconds=TIMEOUT_SECONDS,
wait_for_deployment=True,
insert_strict_options=True):
start = time.time()
# If the package is already installed at this point, fail immediately.
if sdk_marathon.app_exists(service_name):
raise dcos.errors.DCOSException('Service is already installed: {}'.format(service_name))
if insert_strict_options and sdk_utils.is_strict_mode():
# strict mode requires correct principal and secret to perform install.
# see also: sdk_security.py
options = merge_dictionaries({
'service': {
'service_account': 'service-acct',
'principal': 'service-acct',
'service_account_secret': 'secret',
'secret_name': 'secret'
}
}, additional_options)
else:
options = additional_options
# 1. Install package, wait for tasks, wait for marathon deployment
_retried_install_impl(
package_name,
service_name,
expected_running_tasks,
options,
package_version,
timeout_seconds)
# 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit)
# This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete
# state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app)
if wait_for_deployment:
# this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected
# total task count via FINISHED tasks, without actually completing deployment
log.info('Waiting for package={} service={} to finish deployment plan...'.format(
package_name, service_name))
sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds)
log.info('Installed package={} service={} after {}'.format(
package_name, service_name, shakedown.pretty_duration(time.time() - start)))
global _installed_service_names
_installed_service_names.add(service_name)
def zookeeper_service(configure_security):
service_options = sdk_utils.merge_dictionaries(
sdk_networks.ENABLE_VIRTUAL_NETWORKS_OPTIONS,
{"service": {
"name": config.ZOOKEEPER_SERVICE_NAME
}},
)
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
if sdk_utils.is_strict_mode():
service_options = sdk_utils.merge_dictionaries(
{
"service": {
"service_account": zk_account,
"service_account_secret": zk_secret
}
},
service_options,
)
service_account_info = sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
linux_user="******",
service_account=zk_account,
service_account_secret=zk_secret,
)
sdk_install.install(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False,
)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
}
}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME,
service_account_info)
def _get_spark_options(service_name, additional_options):
options = {"service": {"user": "******", "name": service_name}}
if sdk_utils.is_strict_mode():
# At the moment, we do this by hand because Spark doesn't quite line up with other services
# with these options, and sdk_install assumes that we're following those conventions
# Specifically, Spark's config.json lacks: service.principal, service.secret_name
options["service"]["service_account"] = SPARK_SERVICE_ACCOUNT
options["service"][
"service_account_secret"] = SPARK_SERVICE_ACCOUNT_SECRET
return sdk_install.merge_dictionaries(options, additional_options)
def zookeeper_server(configure_security):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"virtual_network_enabled": True
}
}
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
if sdk_utils.is_strict_mode():
service_options = sdk_install.merge_dictionaries({
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
sdk_security.setup_security(config.ZOOKEEPER_SERVICE_NAME, zk_account, zk_secret)
sdk_install.install(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {**service_options, **{"package_name": config.ZOOKEEPER_PACKAGE_NAME}}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
if sdk_utils.is_strict_mode():
sdk_security.delete_service_account(
service_account_name=zk_account, service_account_secret=zk_secret)
def zookeeper_server(kerberos):
service_kerberos_options = {
"service": {
"name": "kafka-zookeeper",
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": sdk_auth.REALM,
"keytab_secret": kerberos.get_keytab_path(),
}
}
}
}
zk_account = "kafka-zookeeper-service-account"
zk_secret = "kakfa-zookeeper-secret"
if sdk_utils.is_strict_mode():
service_kerberos_options = sdk_install.merge_dictionaries(
{
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_kerberos_options)
try:
sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper")
sdk_security.setup_security("kafka-zookeeper", zk_account, zk_secret)
sdk_install.install("beta-kafka-zookeeper",
"kafka-zookeeper",
6,
additional_options=service_kerberos_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {
**service_kerberos_options,
**{
"package_name": "beta-kafka-zookeeper"
}
}
finally:
sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper")
def get_package_options(additional_options={}):
# expected SECURITY values: 'permissive', 'strict', 'disabled'
if sdk_utils.is_strict_mode():
# strict mode requires correct principal and secret to perform install.
# see also: sdk_security.py
return merge_dictionaries({
'service': {
'service_account': 'service-acct',
'principal': 'service-acct',
'service_account_secret': 'secret',
'secret_name': 'secret'
}
}, additional_options)
else:
return additional_options
def zookeeper_server(configure_security):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"virtual_network_enabled": True
}
}
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
if sdk_utils.is_strict_mode():
service_options = sdk_install.merge_dictionaries(
{
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
service_account_info = sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
linux_user="******",
service_account=zk_account,
service_account_secret=zk_secret)
sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
}
}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME,
service_account_info)
def security_session(framework_name: str) -> None:
"""Create a service account and configure permissions for strict-mode tests.
This should generally be used as a fixture in a framework's conftest.py:
@pytest.fixture(scope='session')
def configure_security(configure_universe):
yield from sdk_security.security_session(framework_name)
"""
try:
is_strict = sdk_utils.is_strict_mode()
if is_strict:
setup_security(framework_name)
yield
finally:
if is_strict:
cleanup_security(framework_name)
def cleanup_security(framework_name: str,
service_account: str = "service-acct",
service_account_secret: str = "secret") -> None:
if sdk_utils.is_strict_mode():
log.info("Cleaning up strict-mode security")
revoke_permissions(linux_user="******",
role_name="{}-role".format(framework_name),
service_account_name=service_account)
revoke_permissions(
linux_user="******",
role_name="test__integration__{}-role".format(framework_name),
service_account_name=service_account)
delete_service_account(service_account, service_account_secret)
log.info("Finished cleaning up strict-mode security")
def get_package_options(additional_options={}):
# expected SECURITY values: 'permissive', 'strict', 'disabled'
if sdk_utils.is_strict_mode():
# strict mode requires correct principal and secret to perform install.
# see also: tools/setup_permissions.sh and tools/create_service_account.sh
return merge_dictionaries(
{
'service': {
'service_account': 'service-acct',
'principal': 'service-acct',
'service_account_secret': 'secret',
'secret_name': 'secret',
'mesos_api_version': 'V0'
}
}, additional_options)
else:
return additional_options
def zookeeper_server(kerberos):
service_kerberos_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": sdk_auth.REALM,
"keytab_secret": kerberos.get_keytab_path(),
}
}
}
}
zk_account = "kafka-zookeeper-service-account"
zk_secret = "kakfa-zookeeper-secret"
if sdk_utils.is_strict_mode():
service_kerberos_options = sdk_install.merge_dictionaries({
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_kerberos_options)
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
sdk_security.setup_security(config.ZOOKEEPER_SERVICE_NAME, zk_account, zk_secret)
sdk_install.install(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_kerberos_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {**service_kerberos_options, **{"package_name": config.ZOOKEEPER_PACKAGE_NAME}}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
def _create_service_account(service_name) -> None:
if sdk_utils.is_strict_mode():
try:
log.info("Creating service accounts for '{}'"
.format(service_name))
sa_name = "{}-principal".format(service_name)
sa_secret = "{}-secret".format(service_name)
return sdk_security.setup_security(
service_name,
linux_user="******",
service_account=sa_name,
service_account_secret=sa_secret,
)
except Exception as e:
log.warning("Error encountered while creating service account: {}".format(e))
raise e
return None
def setup_security(
service_name: str,
roles: List[str] = [],
permissions: List[Dict[str, str]] = [],
linux_user: str = DEFAULT_LINUX_USER,
service_account: str = "service-acct",
service_account_secret: str = "secret",
) -> Dict[str, Any]:
create_service_account(
service_account_name=service_account, service_account_secret=service_account_secret
)
security_info: Dict[str, Any] = {
"name": service_account,
"secret": service_account_secret,
"linux_user": linux_user,
"roles": [],
"permissions": {},
"is_strict": sdk_utils.is_strict_mode(),
}
if not security_info["is_strict"]:
log.info("Skipping strict-mode security setup on non-strict cluster")
return security_info
log.info("Setting up strict-mode security")
security_info["roles"] = roles.copy() if roles else _get_service_role(service_name)
for role_name in security_info["roles"]:
security_info["permissions"][role_name] = grant_permissions(
linux_user=linux_user,
role_name=role_name,
service_account_name=service_account,
permissions=permissions,
)
log.info("Finished setting up strict-mode security")
return security_info
import pytest
import retrying
import shakedown
import sdk_install
import sdk_utils
import sdk_marathon
import sdk_cmd
from tests import config
pytestmark = pytest.mark.skipif(
sdk_utils.is_strict_mode() and sdk_utils.dcos_version_less_than('1.11'),
reason="secure hierarchical roles are only supported on 1.11+")
pre_reserved_options = {
"service": {
"yaml": "pre-reserved"
}
}
@pytest.fixture(scope='module', autouse=True)
def configure_package(configure_security):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
sdk_install.install(config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_TASK_COUNT,
additional_options=pre_reserved_options)