def test_user_propagation(setup_spark, user, use_ucr_containerizer, use_ucr_for_spark_submit):
dispatcher_task = utils.get_dispatcher_task(service_name=SERVICE_NAME)
_check_task_user(dispatcher_task, user, use_ucr_containerizer)
extra_args = []
if use_ucr_for_spark_submit:
extra_args = ["--conf spark.mesos.containerizer=mesos"]
_submit_job_and_verify_users(user, use_ucr_for_spark_submit, extra_args=extra_args)
def test_user_propagation(setup_spark, user, use_ucr_containerizer,
use_ucr_for_spark_submit):
dispatcher_task = utils.get_dispatcher_task(service_name=SERVICE_NAME)
_check_task_user(dispatcher_task, user, use_ucr_containerizer)
extra_args = []
if not use_ucr_for_spark_submit:
extra_args = ["--conf spark.mesos.containerizer=docker"]
if user == "nobody":
extra_args = extra_args + [
"--conf spark.mesos.executor.docker.parameters=user=99"
]
_submit_job_and_verify_users(user,
use_ucr_for_spark_submit,
extra_args=extra_args)
def test_user_overrides(setup_spark, user, user_override, use_ucr_containerizer, use_ucr_for_spark_submit):
"""This test uses overrides for 'root' user only to lower down total amount of tests
while preserving containerizers combinations. User override functionality is independent
from any specific username, thus submitting jobs to Dispatcher running as 'root' with
jobs running as 'nobody' is sufficient to test it.
"""
dispatcher_task = utils.get_dispatcher_task(service_name=SERVICE_NAME)
_check_task_user(dispatcher_task, user, use_ucr_containerizer)
extra_args = [f"--conf spark.mesos.driverEnv.SPARK_USER={user_override}"]
if use_ucr_for_spark_submit:
extra_args = extra_args + ["--conf spark.mesos.containerizer=mesos"]
else:
extra_args = extra_args + [f"--conf spark.mesos.executor.docker.parameters=user={user_override}"]
_submit_job_and_verify_users(user_override, use_ucr_for_spark_submit, extra_args=extra_args)
def test_dispatcher_default_network(spark_dispatcher, use_ucr_containerizer,
use_ucr_for_spark_submit):
log.info(
"Running test with use_ucr_containerizer={}, use_ucr_for_spark_submit={}"
.format(use_ucr_containerizer, use_ucr_for_spark_submit))
dispatcher_task = utils.get_dispatcher_task(
service_name=CNI_DISPATCHER_SERVICE_NAME)
_check_task_network(dispatcher_task, is_ucr=use_ucr_containerizer)
dispatcher_ip = sdk_networks.get_task_ip(dispatcher_task)
submit_args = [
"--master mesos://{}:7077".format(dispatcher_ip),
"--deploy-mode cluster",
"--conf spark.mesos.executor.docker.image={}".format(
utils.SPARK_DOCKER_IMAGE)
]
test_shuffle_job(submit_args=submit_args,
use_ucr_for_spark_submit=use_ucr_for_spark_submit,
use_cli_for_spark_submit=False)
def test_cni_dispatcher(spark_dispatcher, use_ucr_containerizer):
task = utils.get_dispatcher_task(service_name=CNI_DISPATCHER_SERVICE_NAME)
_check_task_network(task, is_ucr=use_ucr_containerizer)