def install(self, kerberos: sdk_auth.KerberosEnvironment=None) -> dict: options = { "id": self.id, "mem": 512, "container": { "type": "MESOS", "docker": { "image": "elezar/kafka-client:deca3d0", "forcePullImage": True } }, "networks": [ { "mode": "host" } ], "env": { "JVM_MaxHeapSize": "512", "KAFKA_CLIENT_MODE": "test", "KAFKA_TOPIC": "securetest" } } if kerberos is not None: self._is_kerberos = True options = sdk_install.merge_dictionaries(options, self._get_kerberos_options(kerberos)) sdk_marathon.install_app(options) return options
def configure_zookeeper(configure_security, install_zookeeper_stub): service_options = { "service": { "virtual_network_enabled": True } } zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" if sdk_utils.is_strict_mode(): service_options = sdk_install.merge_dictionaries({ 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) try: sdk_install.uninstall(ZK_PACKAGE, ZK_SERVICE_NAME) sdk_security.setup_security(ZK_SERVICE_NAME, zk_account, zk_secret) sdk_install.install( ZK_PACKAGE, ZK_SERVICE_NAME, 6, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield finally: sdk_install.uninstall(ZK_PACKAGE, ZK_SERVICE_NAME)
def get_spark_options(service_name, additional_options): options = {"service": {"user": SPARK_USER, "name": service_name}} if SPARK_DOCKER_USER is not None: options["service"]["docker_user"] = SPARK_DOCKER_USER if sdk_utils.is_strict_mode(): # At the moment, we do this by hand because Spark doesn't quite line up with other services # with these options, and sdk_install assumes that we're following those conventions # Specifically, Spark's config.json lacks: service.principal, service.secret_name options["service"]["service_account"] = SPARK_SERVICE_ACCOUNT options["service"][ "service_account_secret"] = SPARK_SERVICE_ACCOUNT_SECRET print(sdk_install.merge_dictionaries(options, additional_options)) return sdk_install.merge_dictionaries(options, additional_options)
def zookeeper_server(kerberos): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "security": { "kerberos": { "enabled": True, "kdc": { "hostname": kerberos.get_host(), "port": int(kerberos.get_port()) }, "realm": kerberos.get_realm(), "keytab_secret": kerberos.get_keytab_path(), } } } } zk_account = "kafka-zookeeper-service-account" zk_secret = "kakfa-zookeeper-secret" if sdk_utils.is_strict_mode(): service_options = sdk_install.merge_dictionaries( { 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) service_account_info = sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, linux_user="******", service_account=zk_account, service_account_secret=zk_secret) sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME } } finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME, service_account_info)
def merge_service_options(options: typing.List[typing.Dict]) -> typing.Dict: """ Merge the specified service options with the options later in the list taking precedence. """ result = {} for o in options: result = sdk_install.merge_dictionaries(result, o) return result
def zookeeper_server(kerberos): service_kerberos_options = { "service": { "name": "kafka-zookeeper", "security": { "kerberos": { "enabled": True, "kdc": { "hostname": kerberos.get_host(), "port": int(kerberos.get_port()) }, "realm": sdk_auth.REALM, "keytab_secret": kerberos.get_keytab_path(), } } } } zk_account = "kafka-zookeeper-service-account" zk_secret = "kakfa-zookeeper-secret" if sdk_utils.is_strict_mode(): service_kerberos_options = sdk_install.merge_dictionaries( { 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_kerberos_options) try: sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper") sdk_security.setup_security("kafka-zookeeper", zk_account, zk_secret) sdk_install.install("beta-kafka-zookeeper", "kafka-zookeeper", 6, additional_options=service_kerberos_options, timeout_seconds=30 * 60, insert_strict_options=False) yield { **service_kerberos_options, **{ "package_name": "beta-kafka-zookeeper" } } finally: sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper")
def get_strict_mode_options(service_account_info: typing.Dict) -> typing.Dict: options = {} if "linux_user" in service_account_info: user_options = { "service": { "user": service_account_info["linux_user"] } } options = sdk_install.merge_dictionaries(options, user_options) if sdk_utils.is_strict_mode(): service_account_options = { 'service': { 'service_account': service_account_info["name"], 'service_account_secret': service_account_info["secret"], } } options = sdk_install.merge_dictionaries(options, service_account_options) return options
def zookeeper_server(configure_security): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "virtual_network_enabled": True } } zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) if sdk_utils.is_strict_mode(): service_options = sdk_install.merge_dictionaries( { 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) service_account_info = sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, linux_user="******", service_account=zk_account, service_account_secret=zk_secret) sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME } } finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME, service_account_info)
def install(package_name, service_name, expected_running_tasks, additional_options={}, package_version=None, timeout_seconds=25 * 60, wait_for_deployment=True): test_options = {"brokers": {"cpus": 0.5}} sdk_install.install(package_name=package_name, expected_running_tasks=expected_running_tasks, service_name=service_name, additional_options=sdk_install.merge_dictionaries( test_options, additional_options), package_version=package_version, timeout_seconds=timeout_seconds, wait_for_deployment=wait_for_deployment)
def zookeeper_server(kerberos): service_kerberos_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "security": { "kerberos": { "enabled": True, "kdc": { "hostname": kerberos.get_host(), "port": int(kerberos.get_port()) }, "realm": sdk_auth.REALM, "keytab_secret": kerberos.get_keytab_path(), } } } } zk_account = "kafka-zookeeper-service-account" zk_secret = "kakfa-zookeeper-secret" if sdk_utils.is_strict_mode(): service_kerberos_options = sdk_install.merge_dictionaries({ 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_kerberos_options) try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.setup_security(config.ZOOKEEPER_SERVICE_NAME, zk_account, zk_secret) sdk_install.install( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_kerberos_options, timeout_seconds=30 * 60, insert_strict_options=False) yield {**service_kerberos_options, **{"package_name": config.ZOOKEEPER_PACKAGE_NAME}} finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
def spark_dispatcher(configure_security_spark, configure_universe, use_ucr_containerizer): utils.teardown_spark(service_name=CNI_DISPATCHER_SERVICE_NAME, zk=CNI_DISPATCHER_ZK) options = {"service": {"UCR_containerizer": use_ucr_containerizer}} try: merged_options = sdk_install.merge_dictionaries( CNI_SERVICE_OPTIONS, options) sdk_install.install(utils.SPARK_PACKAGE_NAME, CNI_DISPATCHER_SERVICE_NAME, 0, additional_options=utils.get_spark_options( CNI_DISPATCHER_SERVICE_NAME, merged_options), wait_for_deployment=False) yield finally: utils.teardown_spark(service_name=CNI_DISPATCHER_SERVICE_NAME, zk=CNI_DISPATCHER_ZK)
def install( package_name, service_name, expected_running_tasks, additional_options={}, package_version=None, timeout_seconds=25*60, wait_for_deployment=True): test_options = { "brokers": { "cpus": 0.5 } } sdk_install.install(package_name=package_name, expected_running_tasks=expected_running_tasks, service_name=service_name, additional_options=sdk_install.merge_dictionaries(test_options, additional_options), package_version=package_version, timeout_seconds=timeout_seconds, wait_for_deployment=wait_for_deployment)
def zookeeper_server(configure_security): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "virtual_network_enabled": True } } zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) if sdk_utils.is_strict_mode(): service_options = sdk_install.merge_dictionaries({ 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) sdk_security.setup_security(config.ZOOKEEPER_SERVICE_NAME, zk_account, zk_secret) sdk_install.install( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield {**service_options, **{"package_name": config.ZOOKEEPER_PACKAGE_NAME}} finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) if sdk_utils.is_strict_mode(): sdk_security.delete_service_account( service_account_name=zk_account, service_account_secret=zk_secret)