def zookeeper_server(configure_security): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "virtual_network_enabled": True, } } zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" try: sdk_install.uninstall( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, ) if sdk_utils.is_strict_mode(): service_options = sdk_utils.merge_dictionaries( { "service": { "service_account": zk_account, "service_account_secret": zk_secret, } }, service_options, ) sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, service_account=zk_account, service_account_secret=zk_secret, ) sdk_install.install( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, package_version=config.ZOOKEEPER_PACKAGE_VERSION, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False, ) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME }, } finally: sdk_install.uninstall( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, ) if sdk_utils.is_strict_mode(): sdk_security.delete_service_account( service_account_name=zk_account, service_account_secret=zk_secret, )
def _submit_consumer(name, spark_executor_docker_image, jar, kafka_broker_dns, cassandra_native_client_dns, dispatcher, kafka_topics, kafka_group_id, write_to_cassandra, batch_size_seconds, cassandra_keyspace, cassandra_table, spark_cores_max, spark_executor_cores, must_fail: bool): app_args = ["--appName", name, "--brokers", ",".join(kafka_broker_dns), "--topics", kafka_topics, "--groupId", kafka_group_id, "--batchSizeSeconds", str(batch_size_seconds), "--cassandraKeyspace", cassandra_keyspace, "--cassandraTable", cassandra_table] if must_fail: app_args.extend(["--mustFailDueToInvalidArgument"]) if not write_to_cassandra: app_args.extend(["--shouldNotWriteToCassandra"]) cassandra_hosts = map(lambda x: x.split(':')[0], cassandra_native_client_dns) cassandra_port = cassandra_native_client_dns[0].split(':')[1] app_config = ["--supervise", "--conf", "spark.cores.max={}".format(spark_cores_max), "--conf", "spark.executor.cores={}".format(spark_executor_cores), "--conf", "spark.cassandra.connection.host={}".format(",".join(cassandra_hosts)), "--conf", "spark.cassandra.connection.port={}".format(cassandra_port), "--name", name, "--class", CONSUMER_CLASS_NAME] if spark_executor_docker_image: app_config.extend([ "--conf", "spark.mesos.executor.docker.image={}".format(spark_executor_docker_image) ]) args = app_config + COMMON_CONF submission_id = spark_utils.submit_job( app_url=jar, app_args=" ".join(str(a) for a in app_args), args=args, verbose=False, service_name=dispatcher['service']['name'], driver_role=dispatcher['roles']['executors'], spark_user=dispatcher['service']['user'] if sdk_utils.is_strict_mode() else None, principal=dispatcher['service']['service_account'] if sdk_utils.is_strict_mode() else None) return submission_id
def configure_role_permissions(configure_security_spark): try: if sdk_utils.is_strict_mode(): utils.grant_user_permissions("nobody", driver_role, utils.SPARK_SERVICE_ACCOUNT) utils.grant_launch_task_permission(service_name) yield finally: if sdk_utils.is_strict_mode(): utils.revoke_user_permissions("nobody", driver_role, utils.SPARK_SERVICE_ACCOUNT) utils.revoke_launch_task_permission(service_name)
def submit_job(app_url: str, app_args: str, dispatcher: typing.Dict, duration: int, config: typing.List[str]): dispatcher_name = dispatcher["service"]["name"] log.info("Submitting job to dispatcher: %s, with duration: %s min.", dispatcher_name, duration) spark_utils.submit_job( service_name=dispatcher_name, app_url=app_url, app_args=app_args, verbose=False, args=config, driver_role=dispatcher["roles"]["executors"], spark_user=dispatcher["service"]["user"] if sdk_utils.is_strict_mode() else None, principal=dispatcher["service"]["service_account"] if sdk_utils.is_strict_mode() else None)
def spark_security_session(): ''' Spark strict mode setup is slightly different from dcos-commons, so can't use sdk_security::security_session. Differences: (1) the role is "*", (2) the driver itself is a framework and needs permission to execute tasks. ''' user = SPARK_USER role = '*' service_account = SPARK_SERVICE_ACCOUNT secret = SPARK_SERVICE_ACCOUNT_SECRET def grant_driver_permission(service_account_name, service_name): app_id = "/{}".format(service_name.lstrip("/")) # double-encoded (why?) app_id = urllib.parse.quote(urllib.parse.quote(app_id, safe=''), safe='') sdk_security._grant( service_account_name, "dcos:mesos:master:task:app_id:{}".format(app_id), description="Spark drivers may execute Mesos tasks", action="create") def setup_security(): LOGGER.info('Setting up strict-mode security for Spark') sdk_security.create_service_account( service_account_name=service_account, service_account_secret=secret) sdk_security.grant_permissions(linux_user=user, role_name=role, service_account_name=service_account) grant_driver_permission(service_account, SPARK_SERVICE_NAME) grant_driver_permission(service_account, FOLDERED_SPARK_SERVICE_NAME) LOGGER.info('Finished setting up strict-mode security for Spark') def cleanup_security(): LOGGER.info('Cleaning up strict-mode security for Spark') sdk_security.revoke_permissions(linux_user=user, role_name=role, service_account_name=service_account) sdk_security.delete_service_account(service_account, secret) LOGGER.info('Finished cleaning up strict-mode security for Spark') try: if sdk_utils.is_strict_mode(): setup_security() yield finally: if sdk_utils.is_strict_mode(): cleanup_security()
def configure_user_permissions(configure_security_spark): """Tests in this module require permissions for 'root' user, we can't use `spark_utils.spark_security_session` here because it drops service account completely during teardown. To lower down amount of boilerplate code in tests 'configure_security_spark' fixture is defined with 'session' scope thus dropped service account won't be recreated in tests executed after the current one. """ try: if sdk_utils.is_strict_mode(): utils.grant_launch_task_permission(SERVICE_NAME) utils.grant_user_permissions("root") yield finally: if sdk_utils.is_strict_mode(): utils.revoke_launch_task_permission(SERVICE_NAME) utils.revoke_user_permissions("root")
def _submit_producer(name, spark_executor_docker_image, jar, kafka_broker_dns, dispatcher, kafka_topics, number_of_words, words_per_second, spark_cores_max, spark_executor_cores, must_fail: bool): app_args = ["--appName", name, "--brokers", ",".join(kafka_broker_dns), "--topics", kafka_topics, "--numberOfWords", str(number_of_words), "--wordsPerSecond", str(words_per_second)] if must_fail: app_args.extend(["--mustFailDueToInvalidArgument", ]) app_config = ["--conf", "spark.cores.max={}".format(spark_cores_max), "--conf", "spark.executor.cores={}".format(spark_executor_cores), "--name", name, "--class", PRODUCER_CLASS_NAME] # `number_of_words == 0` means infinite stream, so we'd like to have it # restarted in the case of failures. if number_of_words == 0: app_config.extend(["--supervise"]) if spark_executor_docker_image: app_config.extend([ "--conf", "spark.mesos.executor.docker.image={}".format(spark_executor_docker_image) ]) args = app_config + COMMON_CONF submission_id = spark_utils.submit_job( app_url=jar, app_args=" ".join(str(a) for a in app_args), args=args, verbose=False, service_name=dispatcher['service']['name'], driver_role=dispatcher['roles']['executors'], spark_user=dispatcher['service']['user'] if sdk_utils.is_strict_mode() else None, principal=dispatcher['service']['service_account'] if sdk_utils.is_strict_mode() else None) return submission_id
def submit_job(app_url, app_args, service_name=SPARK_SERVICE_NAME, args=[], spark_user=None, driver_role=SPARK_DRIVER_ROLE, verbose=True, principal=SPARK_SERVICE_ACCOUNT, use_cli=True): conf_args = args.copy() if driver_role: conf_args += ['--conf', 'spark.mesos.role={}'.format(driver_role)] if SPARK_DOCKER_USER is not None: conf_args += [ '--conf', 'spark.mesos.executor.docker.parameters=user={}'.format( SPARK_DOCKER_USER) ] if not list(filter(lambda x: "spark.driver.memory=" in x, conf_args)): conf_args += ['--conf', 'spark.driver.memory=2g'] if sdk_utils.is_strict_mode(): conf_args += ['--conf spark.mesos.principal={}'.format(principal)] if spark_user is not None: conf_args += [ '--conf spark.mesos.driverEnv.SPARK_USER={}'.format(spark_user) ] if not list(filter(lambda x: "spark.mesos.containerizer=" in x, conf_args)): conf_args += ['--conf', 'spark.mesos.containerizer=mesos'] submit_args = ' '.join([' '.join(conf_args), app_url, app_args]) verbose_flag = "--verbose" if verbose else "" result = None if use_cli: stdout = sdk_cmd.svc_cli( SPARK_PACKAGE_NAME, service_name, 'run {} --submit-args="{}"'.format(verbose_flag, submit_args)) result = re.search(r"Submission id: (\S+)", stdout) else: docker_cmd = "sudo docker run --net=host -ti {} bin/spark-submit {}".format( SPARK_DOCKER_IMAGE, submit_args) ssh_opts = "--option UserKnownHostsFile=/dev/null --option StrictHostKeyChecking=no" log.info("Running Docker command on leader: {}".format(docker_cmd)) _, stdout, stderr = sdk_cmd.run_raw_cli( "node ssh --master-proxy --leader --user={} {} '{}'".format( sdk_cmd.LINUX_USER, ssh_opts, docker_cmd)) result = re.search(r'"submissionId" : "(\S+)"', stdout) if not result: raise Exception( "Unable to find submission ID in stdout:\n{}".format(stdout)) return result.group(1)
def security_session( framework_name: str, permissions: List[Dict[str, str]] = [], linux_user: str = DEFAULT_LINUX_USER, service_account: str = "service-acct", service_account_secret: str = "secret", ) -> Iterator[None]: """Create a service account and configure permissions for strict-mode tests. This should generally be used as a fixture in a framework's conftest.py: @pytest.fixture(scope='session') def configure_security(configure_universe): yield from sdk_security.security_session(framework_name, permissions, linux_user, 'service-acct') """ is_strict = sdk_utils.is_strict_mode() try: if is_strict: roles = _get_service_role(framework_name) + _get_integration_test_foldered_role( framework_name ) security_info = setup_security( framework_name, roles=roles, permissions=permissions, linux_user=linux_user, service_account=service_account, service_account_secret=service_account_secret, ) yield finally: if is_strict: cleanup_security(framework_name, security_info)
def kerberized_kafka(configure_security_kafka, kerberos_options): try: additional_options = { "service": { "name": KAFKA_SERVICE_NAME, "security": kerberos_options }, "kafka": { "default_replication_factor": 3, "num_partitions": 32 } } if sdk_utils.is_strict_mode(): additional_options["service"][ "service_account"] = KAFKA_SERVICE_ACCOUNT additional_options["service"]["principal"] = KAFKA_SERVICE_ACCOUNT additional_options["service"][ "service_account_secret"] = KAFKA_SERVICE_ACCOUNT_SECRET additional_options["service"][ "secret_name"] = KAFKA_SERVICE_ACCOUNT_SECRET sdk_install.uninstall(KAFKA_PACKAGE_NAME, KAFKA_SERVICE_NAME) sdk_install.install(KAFKA_PACKAGE_NAME, KAFKA_SERVICE_NAME, DEFAULT_KAFKA_TASK_COUNT, additional_options=additional_options, timeout_seconds=30 * 60) yield finally: sdk_install.uninstall(KAFKA_PACKAGE_NAME, KAFKA_SERVICE_NAME)
def configure_zookeeper(configure_security, install_zookeeper_stub): service_options = { "service": { "virtual_network_enabled": True } } zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" if sdk_utils.is_strict_mode(): service_options = sdk_install.merge_dictionaries({ 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) try: sdk_install.uninstall(ZK_PACKAGE, ZK_SERVICE_NAME) sdk_security.setup_security(ZK_SERVICE_NAME, zk_account, zk_secret) sdk_install.install( ZK_PACKAGE, ZK_SERVICE_NAME, 6, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield finally: sdk_install.uninstall(ZK_PACKAGE, ZK_SERVICE_NAME)
def hdfs_with_kerberos(configure_security_hdfs, kerberos_options): try: additional_options = { "service": { "name": HDFS_SERVICE_NAME, "security": kerberos_options }, "hdfs": { "security_auth_to_local": hdfs_auth.get_principal_to_user_mapping() } } if sdk_utils.is_strict_mode(): additional_options["service"][ "service_account"] = HDFS_SERVICE_ACCOUNT additional_options["service"]["principal"] = HDFS_SERVICE_ACCOUNT additional_options["service"][ "service_account_secret"] = HDFS_SERVICE_ACCOUNT_SECRET additional_options["service"][ "secret_name"] = HDFS_SERVICE_ACCOUNT_SECRET sdk_install.uninstall(HDFS_PACKAGE_NAME, HDFS_SERVICE_NAME) sdk_install.install(HDFS_PACKAGE_NAME, HDFS_SERVICE_NAME, DEFAULT_HDFS_TASK_COUNT, additional_options=additional_options, timeout_seconds=30 * 60) yield finally: sdk_install.uninstall(HDFS_PACKAGE_NAME, HDFS_SERVICE_NAME)
def configure_role_permissions(configure_security_spark): try: if sdk_utils.is_strict_mode(): for role in ROLES: utils.grant_user_permissions("nobody", role, utils.SPARK_SERVICE_ACCOUNT) utils.grant_launch_task_permission("/dev/spark") yield finally: if sdk_utils.is_strict_mode(): for role in ROLES: utils.revoke_user_permissions("nobody", role, utils.SPARK_SERVICE_ACCOUNT) utils.revoke_launch_task_permission("/dev/spark")
def security_session( framework_name: str, permissions: List[Dict[str, str]] = [], linux_user: str = DEFAULT_LINUX_USER, service_account: str = "service-acct", service_account_secret: str = "secret", ) -> Iterator[None]: """Create a service account and configure permissions for strict-mode tests. This should generally be used as a fixture in a framework's conftest.py: @pytest.fixture(scope='session') def configure_security(configure_universe): yield from sdk_security.security_session(framework_name, permissions, linux_user, 'service-acct') """ is_strict = sdk_utils.is_strict_mode() try: if is_strict: roles = _get_service_role( framework_name) + _get_integration_test_foldered_role( framework_name) security_info = setup_security( framework_name, roles=roles, permissions=permissions, linux_user=linux_user, service_account=service_account, service_account_secret=service_account_secret, ) yield finally: if is_strict: cleanup_security(framework_name, security_info)
def setup_security(framework_name: str, service_account: str = "service-acct", service_account_secret: str = "secret") -> dict: create_service_account(service_account_name=service_account, service_account_secret=service_account_secret) service_account_info = { "name": service_account, "secret": service_account_secret } if not sdk_utils.is_strict_mode(): log.info("Skipping strict-mode security setup on non-strict cluster") return service_account_info log.info("Setting up strict-mode security") grant_permissions(linux_user="******", role_name="{}-role".format(framework_name), service_account_name=service_account) grant_permissions( linux_user="******", role_name="slave_public%252F{}-role".format(framework_name), service_account_name=service_account) grant_permissions( linux_user="******", role_name="test__integration__{}-role".format(framework_name), service_account_name=service_account) log.info("Finished setting up strict-mode security") return service_account_info
def submit_job(app_url, app_args, service_name=SPARK_SERVICE_NAME, args=[], spark_user=SPARK_USER, driver_role=SPARK_DRIVER_ROLE, verbose=True, principal=SPARK_SERVICE_ACCOUNT): conf_args = args.copy() conf_args += ['--conf', 'spark.mesos.role={}'.format(driver_role)] if not list( filter(lambda x: x.startswith("spark.driver.memory="), conf_args)): conf_args += ['--conf', 'spark.driver.memory=2g'] if sdk_utils.is_strict_mode(): conf_args += [ '--conf spark.mesos.driverEnv.SPARK_USER={}'.format(spark_user), '--conf spark.mesos.principal={}'.format(principal) ] submit_args = ' '.join([' '.join(conf_args), app_url, app_args]) verbose_flag = "--verbose" if verbose else "" stdout = sdk_cmd.svc_cli( SPARK_PACKAGE_NAME, service_name, 'run {} --submit-args="{}"'.format(verbose_flag, submit_args)) result = re.search(r"Submission id: (\S+)", stdout) if not result: raise Exception( "Unable to find submission ID in stdout:\n{}".format(stdout)) return result.group(1)
def setup_security(service_name: str, linux_user: str="nobody", service_account: str="service-acct", service_account_secret: str="secret") -> dict: create_service_account(service_account_name=service_account, service_account_secret=service_account_secret) service_account_info = {"name": service_account, "secret": service_account_secret, "linux_user": linux_user, "roles": [] } if not sdk_utils.is_strict_mode(): log.info("Skipping strict-mode security setup on non-strict cluster") return service_account_info log.info("Setting up strict-mode security") service_account_info["roles"] = _get_role_list(service_name) for role_name in service_account_info["roles"]: grant_permissions( linux_user=linux_user, role_name=role_name, service_account_name=service_account ) log.info("Finished setting up strict-mode security") return service_account_info
def spark_security_session( users=[SPARK_USER], service_names=[SPARK_SERVICE_NAME, FOLDERED_SPARK_SERVICE_NAME]): ''' Spark strict mode setup is slightly different from dcos-commons, so can't use sdk_security::security_session. Differences: (1) the role is "*", (2) the driver itself is a framework and needs permission to execute tasks. ''' role = '*' service_account = SPARK_SERVICE_ACCOUNT secret = SPARK_SERVICE_ACCOUNT_SECRET def setup_security(): log.info('Setting up strict-mode security for Spark') sdk_security.create_service_account( service_account_name=service_account, service_account_secret=secret) for user in users: grant_user_permissions(user, role, service_account) for service_name in service_names: grant_launch_task_permission(service_name) log.info('Finished setting up strict-mode security for Spark') def cleanup_security(): log.info('Cleaning up strict-mode security for Spark') for user in users: revoke_user_permissions(user, role, service_account) # TODO: improve security setup/teardown to make it more fine-grained (allow different service names/accts/users) # tracking issue: https://jira.mesosphere.com/browse/DCOS-50933 sdk_security.delete_service_account(service_account, secret) log.info('Finished cleaning up strict-mode security for Spark') try: if not sdk_utils.is_open_dcos(): sdk_security.install_enterprise_cli() if sdk_utils.is_strict_mode(): setup_security() yield finally: if sdk_utils.is_strict_mode(): cleanup_security()
def install( package_name: str, service_name: str, expected_running_tasks: int, additional_options: dict = {}, package_version: PackageVersion = PackageVersion.STUB_UNIVERSE, timeout_seconds: int = TIMEOUT_SECONDS, wait_for_deployment: bool = True, insert_strict_options: bool = True, wait_for_all_conditions: bool = True, ) -> None: start = time.time() # If the package is already installed at this point, fail immediately. if sdk_marathon.app_exists(service_name): raise Exception( "Service is already installed: {}".format(service_name)) if insert_strict_options and sdk_utils.is_strict_mode(): # strict mode requires correct principal and secret to perform install. # see also: sdk_security.py options = sdk_utils.merge_dictionaries( { "service": { "service_account": "service-acct", "principal": "service-acct", "service_account_secret": "secret", "secret_name": "secret", } }, additional_options, ) else: options = additional_options # 1. Install package, wait for tasks, wait for marathon deployment _retried_install_impl( package_name, service_name, expected_running_tasks, package_version.value if isinstance( package_version, PackageVersion) else package_version, options, timeout_seconds, wait_for_all_conditions) # 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit) # This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete # state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app) if wait_for_deployment: # this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected # total task count via FINISHED tasks, without actually completing deployment log.info( "Waiting for package={} service={} to finish deployment plan...". format(package_name, service_name)) sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds) log.info("Installed package={} service={} after {}".format( package_name, service_name, sdk_utils.pretty_duration(time.time() - start))) global _installed_service_names _installed_service_names.add(service_name)
def zookeeper_server(kerberos): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "security": { "kerberos": { "enabled": True, "kdc": { "hostname": kerberos.get_host(), "port": int(kerberos.get_port()) }, "realm": kerberos.get_realm(), "keytab_secret": kerberos.get_keytab_path(), } } } } zk_account = "kafka-zookeeper-service-account" zk_secret = "kakfa-zookeeper-secret" if sdk_utils.is_strict_mode(): service_options = sdk_install.merge_dictionaries( { 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) service_account_info = sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, linux_user="******", service_account=zk_account, service_account_secret=zk_secret) sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME } } finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME, service_account_info)
def test_jar(service_name=utils.SPARK_SERVICE_NAME): master_url = ("https" if sdk_utils.is_strict_mode() else "http") + "://leader.mesos:5050" spark_job_runner_args = '{} dcos \\"*\\" spark:only 2 --auth-token={}'.format( master_url, shakedown.dcos_acs_token()) utils.run_tests(app_url=utils.upload_mesos_test_jar(), app_args=spark_job_runner_args, expected_output="All tests passed", service_name=service_name, args=['--class com.typesafe.spark.test.mesos.framework.runners.SparkJobRunner'])
def install( package_name, service_name, expected_running_tasks, additional_options={}, package_version=None, timeout_seconds=TIMEOUT_SECONDS, wait_for_deployment=True, insert_strict_options=True, install_cli=True): start = time.time() # If the package is already installed at this point, fail immediately. if sdk_marathon.app_exists(service_name): raise dcos.errors.DCOSException('Service is already installed: {}'.format(service_name)) if insert_strict_options and sdk_utils.is_strict_mode(): # strict mode requires correct principal and secret to perform install. # see also: sdk_security.py options = merge_dictionaries({ 'service': { 'service_account': 'service-acct', 'principal': 'service-acct', 'service_account_secret': 'secret', 'secret_name': 'secret' } }, additional_options) else: options = additional_options # 1. Install package, wait for tasks, wait for marathon deployment _retried_install_impl( package_name, service_name, expected_running_tasks, options, package_version, timeout_seconds, install_cli) # 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit) # This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete # state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app) if wait_for_deployment: # this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected # total task count via FINISHED tasks, without actually completing deployment log.info('Waiting for package={} service={} to finish deployment plan...'.format( package_name, service_name)) sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds) log.info('Installed package={} service={} after {}'.format( package_name, service_name, shakedown.pretty_duration(time.time() - start))) global _installed_service_names _installed_service_names.add(service_name)
def setup_security(service_name: str, linux_user: str) -> typing.Dict: """ Adds a service account and secret for the specified service name. """ if not sdk_utils.is_strict_mode(): return {} service_account = normalize_string("{}-service-account".format(service_name)) service_account_secret = "{}-service-account-secret".format(service_name) return sdk_security.setup_security(service_name, linux_user, service_account, service_account_secret)
def install( package_name, service_name, expected_running_tasks, additional_options={}, package_version=None, timeout_seconds=TIMEOUT_SECONDS, wait_for_deployment=True, insert_strict_options=True): start = time.time() # If the package is already installed at this point, fail immediately. if sdk_marathon.app_exists(service_name): raise dcos.errors.DCOSException('Service is already installed: {}'.format(service_name)) if insert_strict_options and sdk_utils.is_strict_mode(): # strict mode requires correct principal and secret to perform install. # see also: sdk_security.py options = merge_dictionaries({ 'service': { 'service_account': 'service-acct', 'principal': 'service-acct', 'service_account_secret': 'secret', 'secret_name': 'secret' } }, additional_options) else: options = additional_options # 1. Install package, wait for tasks, wait for marathon deployment _retried_install_impl( package_name, service_name, expected_running_tasks, options, package_version, timeout_seconds) # 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit) # This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete # state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app) if wait_for_deployment: # this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected # total task count via FINISHED tasks, without actually completing deployment log.info('Waiting for package={} service={} to finish deployment plan...'.format( package_name, service_name)) sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds) log.info('Installed package={} service={} after {}'.format( package_name, service_name, shakedown.pretty_duration(time.time() - start))) global _installed_service_names _installed_service_names.add(service_name)
def zookeeper_service(configure_security): service_options = sdk_utils.merge_dictionaries( sdk_networks.ENABLE_VIRTUAL_NETWORKS_OPTIONS, {"service": { "name": config.ZOOKEEPER_SERVICE_NAME }}, ) zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) if sdk_utils.is_strict_mode(): service_options = sdk_utils.merge_dictionaries( { "service": { "service_account": zk_account, "service_account_secret": zk_secret } }, service_options, ) service_account_info = sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, linux_user="******", service_account=zk_account, service_account_secret=zk_secret, ) sdk_install.install( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False, ) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME } } finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME, service_account_info)
def _get_spark_options(service_name, additional_options): options = {"service": {"user": "******", "name": service_name}} if sdk_utils.is_strict_mode(): # At the moment, we do this by hand because Spark doesn't quite line up with other services # with these options, and sdk_install assumes that we're following those conventions # Specifically, Spark's config.json lacks: service.principal, service.secret_name options["service"]["service_account"] = SPARK_SERVICE_ACCOUNT options["service"][ "service_account_secret"] = SPARK_SERVICE_ACCOUNT_SECRET return sdk_install.merge_dictionaries(options, additional_options)
def zookeeper_server(configure_security): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "virtual_network_enabled": True } } zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) if sdk_utils.is_strict_mode(): service_options = sdk_install.merge_dictionaries({ 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) sdk_security.setup_security(config.ZOOKEEPER_SERVICE_NAME, zk_account, zk_secret) sdk_install.install( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield {**service_options, **{"package_name": config.ZOOKEEPER_PACKAGE_NAME}} finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) if sdk_utils.is_strict_mode(): sdk_security.delete_service_account( service_account_name=zk_account, service_account_secret=zk_secret)
def zookeeper_server(kerberos): service_kerberos_options = { "service": { "name": "kafka-zookeeper", "security": { "kerberos": { "enabled": True, "kdc": { "hostname": kerberos.get_host(), "port": int(kerberos.get_port()) }, "realm": sdk_auth.REALM, "keytab_secret": kerberos.get_keytab_path(), } } } } zk_account = "kafka-zookeeper-service-account" zk_secret = "kakfa-zookeeper-secret" if sdk_utils.is_strict_mode(): service_kerberos_options = sdk_install.merge_dictionaries( { 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_kerberos_options) try: sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper") sdk_security.setup_security("kafka-zookeeper", zk_account, zk_secret) sdk_install.install("beta-kafka-zookeeper", "kafka-zookeeper", 6, additional_options=service_kerberos_options, timeout_seconds=30 * 60, insert_strict_options=False) yield { **service_kerberos_options, **{ "package_name": "beta-kafka-zookeeper" } } finally: sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper")
def get_package_options(additional_options={}): # expected SECURITY values: 'permissive', 'strict', 'disabled' if sdk_utils.is_strict_mode(): # strict mode requires correct principal and secret to perform install. # see also: sdk_security.py return merge_dictionaries({ 'service': { 'service_account': 'service-acct', 'principal': 'service-acct', 'service_account_secret': 'secret', 'secret_name': 'secret' } }, additional_options) else: return additional_options
def zookeeper_server(configure_security): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "virtual_network_enabled": True } } zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) if sdk_utils.is_strict_mode(): service_options = sdk_install.merge_dictionaries( { 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) service_account_info = sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, linux_user="******", service_account=zk_account, service_account_secret=zk_secret) sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME } } finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME, service_account_info)
def security_session(framework_name: str) -> None: """Create a service account and configure permissions for strict-mode tests. This should generally be used as a fixture in a framework's conftest.py: @pytest.fixture(scope='session') def configure_security(configure_universe): yield from sdk_security.security_session(framework_name) """ try: is_strict = sdk_utils.is_strict_mode() if is_strict: setup_security(framework_name) yield finally: if is_strict: cleanup_security(framework_name)
def cleanup_security(framework_name: str, service_account: str = "service-acct", service_account_secret: str = "secret") -> None: if sdk_utils.is_strict_mode(): log.info("Cleaning up strict-mode security") revoke_permissions(linux_user="******", role_name="{}-role".format(framework_name), service_account_name=service_account) revoke_permissions( linux_user="******", role_name="test__integration__{}-role".format(framework_name), service_account_name=service_account) delete_service_account(service_account, service_account_secret) log.info("Finished cleaning up strict-mode security")
def get_package_options(additional_options={}): # expected SECURITY values: 'permissive', 'strict', 'disabled' if sdk_utils.is_strict_mode(): # strict mode requires correct principal and secret to perform install. # see also: tools/setup_permissions.sh and tools/create_service_account.sh return merge_dictionaries( { 'service': { 'service_account': 'service-acct', 'principal': 'service-acct', 'service_account_secret': 'secret', 'secret_name': 'secret', 'mesos_api_version': 'V0' } }, additional_options) else: return additional_options
def zookeeper_server(kerberos): service_kerberos_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "security": { "kerberos": { "enabled": True, "kdc": { "hostname": kerberos.get_host(), "port": int(kerberos.get_port()) }, "realm": sdk_auth.REALM, "keytab_secret": kerberos.get_keytab_path(), } } } } zk_account = "kafka-zookeeper-service-account" zk_secret = "kakfa-zookeeper-secret" if sdk_utils.is_strict_mode(): service_kerberos_options = sdk_install.merge_dictionaries({ 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_kerberos_options) try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.setup_security(config.ZOOKEEPER_SERVICE_NAME, zk_account, zk_secret) sdk_install.install( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_kerberos_options, timeout_seconds=30 * 60, insert_strict_options=False) yield {**service_kerberos_options, **{"package_name": config.ZOOKEEPER_PACKAGE_NAME}} finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
def _create_service_account(service_name) -> None: if sdk_utils.is_strict_mode(): try: log.info("Creating service accounts for '{}'" .format(service_name)) sa_name = "{}-principal".format(service_name) sa_secret = "{}-secret".format(service_name) return sdk_security.setup_security( service_name, linux_user="******", service_account=sa_name, service_account_secret=sa_secret, ) except Exception as e: log.warning("Error encountered while creating service account: {}".format(e)) raise e return None
def setup_security( service_name: str, roles: List[str] = [], permissions: List[Dict[str, str]] = [], linux_user: str = DEFAULT_LINUX_USER, service_account: str = "service-acct", service_account_secret: str = "secret", ) -> Dict[str, Any]: create_service_account( service_account_name=service_account, service_account_secret=service_account_secret ) security_info: Dict[str, Any] = { "name": service_account, "secret": service_account_secret, "linux_user": linux_user, "roles": [], "permissions": {}, "is_strict": sdk_utils.is_strict_mode(), } if not security_info["is_strict"]: log.info("Skipping strict-mode security setup on non-strict cluster") return security_info log.info("Setting up strict-mode security") security_info["roles"] = roles.copy() if roles else _get_service_role(service_name) for role_name in security_info["roles"]: security_info["permissions"][role_name] = grant_permissions( linux_user=linux_user, role_name=role_name, service_account_name=service_account, permissions=permissions, ) log.info("Finished setting up strict-mode security") return security_info
import pytest import retrying import shakedown import sdk_install import sdk_utils import sdk_marathon import sdk_cmd from tests import config pytestmark = pytest.mark.skipif( sdk_utils.is_strict_mode() and sdk_utils.dcos_version_less_than('1.11'), reason="secure hierarchical roles are only supported on 1.11+") pre_reserved_options = { "service": { "yaml": "pre-reserved" } } @pytest.fixture(scope='module', autouse=True) def configure_package(configure_security): try: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) sdk_install.install(config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_TASK_COUNT, additional_options=pre_reserved_options)