def zookeeper_service(configure_security): service_options = sdk_utils.merge_dictionaries( sdk_networks.ENABLE_VIRTUAL_NETWORKS_OPTIONS, {"service": { "name": config.ZOOKEEPER_SERVICE_NAME }}, ) zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) if sdk_utils.is_strict_mode(): service_options = sdk_utils.merge_dictionaries( { "service": { "service_account": zk_account, "service_account_secret": zk_secret } }, service_options, ) service_account_info = sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, linux_user="******", service_account=zk_account, service_account_secret=zk_secret, ) sdk_install.install( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False, ) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME } } finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME, service_account_info)
def install(self, kerberos: sdk_auth.KerberosEnvironment = None) -> dict: options = { "id": self.id, "mem": 512, "container": { "type": "MESOS", "docker": { "image": "elezar/kafka-client:deca3d0", "forcePullImage": True }, }, "networks": [{ "mode": "host" }], "env": { "JVM_MaxHeapSize": "512", "KAFKA_CLIENT_MODE": "test", "KAFKA_TOPIC": "securetest", }, } if kerberos is not None: self._is_kerberos = True options = sdk_utils.merge_dictionaries( options, self._get_kerberos_options(kerberos)) sdk_marathon.install_app(options) return options
def install( package_name: str, service_name: str, expected_running_tasks: int, additional_options: dict = {}, package_version: PackageVersion = PackageVersion.STUB_UNIVERSE, timeout_seconds: int = TIMEOUT_SECONDS, wait_for_deployment: bool = True, insert_strict_options: bool = True, wait_for_all_conditions: bool = True, ) -> None: start = time.time() # If the package is already installed at this point, fail immediately. if sdk_marathon.app_exists(service_name): raise Exception( "Service is already installed: {}".format(service_name)) if insert_strict_options and sdk_utils.is_strict_mode(): # strict mode requires correct principal and secret to perform install. # see also: sdk_security.py options = sdk_utils.merge_dictionaries( { "service": { "service_account": "service-acct", "principal": "service-acct", "service_account_secret": "secret", "secret_name": "secret", } }, additional_options, ) else: options = additional_options # 1. Install package, wait for tasks, wait for marathon deployment _retried_install_impl( package_name, service_name, expected_running_tasks, package_version.value if isinstance( package_version, PackageVersion) else package_version, options, timeout_seconds, wait_for_all_conditions) # 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit) # This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete # state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app) if wait_for_deployment: # this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected # total task count via FINISHED tasks, without actually completing deployment log.info( "Waiting for package={} service={} to finish deployment plan...". format(package_name, service_name)) sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds) log.info("Installed package={} service={} after {}".format( package_name, service_name, sdk_utils.pretty_duration(time.time() - start))) global _installed_service_names _installed_service_names.add(service_name)
def zookeeper_server(configure_security): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "virtual_network_enabled": True, } } zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" try: sdk_install.uninstall( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, ) if sdk_utils.is_strict_mode(): service_options = sdk_utils.merge_dictionaries( { "service": { "service_account": zk_account, "service_account_secret": zk_secret, } }, service_options, ) sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, service_account=zk_account, service_account_secret=zk_secret, ) sdk_install.install( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, package_version=config.ZOOKEEPER_PACKAGE_VERSION, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False, ) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME }, } finally: sdk_install.uninstall( config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, ) if sdk_utils.is_strict_mode(): sdk_security.delete_service_account( service_account_name=zk_account, service_account_secret=zk_secret, )
def zookeeper_server(kerberos): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "security": { "kerberos": { "enabled": True, "kdc": { "hostname": kerberos.get_host(), "port": int(kerberos.get_port()) }, "realm": kerberos.get_realm(), "keytab_secret": kerberos.get_keytab_path(), } } } } zk_account = "kafka-zookeeper-service-account" zk_secret = "kakfa-zookeeper-secret" if sdk_utils.is_strict_mode(): service_options = sdk_utils.merge_dictionaries( { 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) service_account_info = sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, linux_user="******", service_account=zk_account, service_account_secret=zk_secret) sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME } } finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME, service_account_info)
def zookeeper_server(configure_security): service_options = { "service": { "name": config.ZOOKEEPER_SERVICE_NAME, "virtual_network_enabled": True } } zk_account = "test-zookeeper-service-account" zk_secret = "test-zookeeper-secret" try: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) if sdk_utils.is_strict_mode(): service_options = sdk_utils.merge_dictionaries( { 'service': { 'service_account': zk_account, 'service_account_secret': zk_secret, } }, service_options) service_account_info = sdk_security.setup_security( config.ZOOKEEPER_SERVICE_NAME, linux_user="******", service_account=zk_account, service_account_secret=zk_secret) sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME, config.ZOOKEEPER_TASK_COUNT, additional_options=service_options, timeout_seconds=30 * 60, insert_strict_options=False) yield { **service_options, **{ "package_name": config.ZOOKEEPER_PACKAGE_NAME } } finally: sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME) sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME, service_account_info)
def install(package_name, service_name, expected_running_tasks, additional_options={}, package_version=None, timeout_seconds=25 * 60, wait_for_deployment=True): test_options = {"brokers": {"cpus": 0.5}} sdk_install.install(package_name=package_name, expected_running_tasks=expected_running_tasks, service_name=service_name, additional_options=sdk_utils.merge_dictionaries( test_options, additional_options), package_version=package_version, timeout_seconds=timeout_seconds, wait_for_deployment=wait_for_deployment)
def kafka_server_with_health_check(request, health_check_method, service_account, kerberos): """ A pytest fixture that installs a kafka service. On teardown, the service is uninstalled. """ service_options = { "service": { "name": config.SERVICE_NAME, "health_check": { "enabled": True, "method": health_check_method, "interval": 60, "delay": 20, "timeout": 60, "grace-period": 30, "max-consecutive-failures": 3, "health-check-topic-prefix": "KafkaHealthCheck" } } } if re.search("SASL", request.param): service_options = sdk_utils.merge_dictionaries( { "service": { "security": { "kerberos": { "enabled": True, "kdc": { "hostname": kerberos.get_host(), "port": int(kerberos.get_port()) }, "realm": kerberos.get_realm(), "keytab_secret": kerberos.get_keytab_path(), } } } }, service_options) if re.search("SSL", request.param): service_options = sdk_utils.merge_dictionaries( { "service": { "service_account": service_account["name"], "service_account_secret": service_account["secret"], "security": { "transport_encryption": { "enabled": True } } } }, service_options) if re.search("^SSL$", request.param): service_options = sdk_utils.merge_dictionaries( { "service": { "security": { "ssl_authentication": { "enabled": True } } } }, service_options) sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME) try: sdk_install.install( config.PACKAGE_NAME, config.SERVICE_NAME, config.DEFAULT_BROKER_COUNT, additional_options=service_options, timeout_seconds=30 * 60, ) yield { **{ "package_name": config.PACKAGE_NAME }, "method": health_check_method, "mode": request.param } finally: sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def install_jmx_configured_cassandra(self_signed_trust_store: bool = True, authentication: bool = True): foldered_name = config.get_foldered_service_name() test_jobs: List[Dict[str, Any]] = [] if authentication: test_jobs = config.get_all_jobs( node_address=config.get_foldered_node_address(), auth=True) else: test_jobs = config.get_all_jobs( node_address=config.get_foldered_node_address()) # destroy/reinstall any prior leftover jobs, so that they don't touch the newly installed service: for job in test_jobs: sdk_jobs.install_job(job) sdk_install.uninstall(config.PACKAGE_NAME, foldered_name) install_jmx_secrets() service_options = { "service": { "name": foldered_name, "jmx": { "enabled": True, "rmi_port": 31198, "password_file": PASSWORD_FILE, "access_file": ACCESS_FILE, "key_store": KEY_STORE, "key_store_password_file": KEY_STORE_PASS, }, } } if self_signed_trust_store: service_options = sdk_utils.merge_dictionaries( { "service": { "jmx": { "add_trust_store": True, "trust_store": TRUST_STORE, "trust_store_password_file": TRUST_STORE_PASS, } } }, service_options, ) if authentication: secret_path = foldered_name + "/" + config.SECRET_VALUE create_secret(secret_value=config.SECRET_VALUE, secret_path=secret_path) service_options = sdk_utils.merge_dictionaries( { "service": { "security": { "authentication": { "enabled": True, "superuser": { "password_secret_path": secret_path }, }, "authorization": { "enabled": True }, } } }, service_options, ) sdk_install.install( config.PACKAGE_NAME, foldered_name, config.DEFAULT_TASK_COUNT, additional_options=service_options, )