def zookeeper_service(configure_security):
service_options = sdk_utils.merge_dictionaries(
sdk_networks.ENABLE_VIRTUAL_NETWORKS_OPTIONS,
{"service": {
"name": config.ZOOKEEPER_SERVICE_NAME
}},
)
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
if sdk_utils.is_strict_mode():
service_options = sdk_utils.merge_dictionaries(
{
"service": {
"service_account": zk_account,
"service_account_secret": zk_secret
}
},
service_options,
)
service_account_info = sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
linux_user="******",
service_account=zk_account,
service_account_secret=zk_secret,
)
sdk_install.install(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False,
)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
}
}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME,
service_account_info)
def install(self, kerberos: sdk_auth.KerberosEnvironment = None) -> dict:
options = {
"id": self.id,
"mem": 512,
"container": {
"type": "MESOS",
"docker": {
"image": "elezar/kafka-client:deca3d0",
"forcePullImage": True
},
},
"networks": [{
"mode": "host"
}],
"env": {
"JVM_MaxHeapSize": "512",
"KAFKA_CLIENT_MODE": "test",
"KAFKA_TOPIC": "securetest",
},
}
if kerberos is not None:
self._is_kerberos = True
options = sdk_utils.merge_dictionaries(
options, self._get_kerberos_options(kerberos))
sdk_marathon.install_app(options)
return options
def install(
package_name: str,
service_name: str,
expected_running_tasks: int,
additional_options: dict = {},
package_version: PackageVersion = PackageVersion.STUB_UNIVERSE,
timeout_seconds: int = TIMEOUT_SECONDS,
wait_for_deployment: bool = True,
insert_strict_options: bool = True,
wait_for_all_conditions: bool = True,
) -> None:
start = time.time()
# If the package is already installed at this point, fail immediately.
if sdk_marathon.app_exists(service_name):
raise Exception(
"Service is already installed: {}".format(service_name))
if insert_strict_options and sdk_utils.is_strict_mode():
# strict mode requires correct principal and secret to perform install.
# see also: sdk_security.py
options = sdk_utils.merge_dictionaries(
{
"service": {
"service_account": "service-acct",
"principal": "service-acct",
"service_account_secret": "secret",
"secret_name": "secret",
}
},
additional_options,
)
else:
options = additional_options
# 1. Install package, wait for tasks, wait for marathon deployment
_retried_install_impl(
package_name, service_name,
expected_running_tasks, package_version.value if isinstance(
package_version, PackageVersion) else package_version, options,
timeout_seconds, wait_for_all_conditions)
# 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit)
# This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete
# state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app)
if wait_for_deployment:
# this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected
# total task count via FINISHED tasks, without actually completing deployment
log.info(
"Waiting for package={} service={} to finish deployment plan...".
format(package_name, service_name))
sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds)
log.info("Installed package={} service={} after {}".format(
package_name, service_name,
sdk_utils.pretty_duration(time.time() - start)))
global _installed_service_names
_installed_service_names.add(service_name)
def zookeeper_server(configure_security):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"virtual_network_enabled": True,
}
}
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
try:
sdk_install.uninstall(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
)
if sdk_utils.is_strict_mode():
service_options = sdk_utils.merge_dictionaries(
{
"service": {
"service_account": zk_account,
"service_account_secret": zk_secret,
}
},
service_options,
)
sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
service_account=zk_account,
service_account_secret=zk_secret,
)
sdk_install.install(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
package_version=config.ZOOKEEPER_PACKAGE_VERSION,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False,
)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
},
}
finally:
sdk_install.uninstall(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
)
if sdk_utils.is_strict_mode():
sdk_security.delete_service_account(
service_account_name=zk_account,
service_account_secret=zk_secret,
)
def zookeeper_server(kerberos):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": kerberos.get_realm(),
"keytab_secret": kerberos.get_keytab_path(),
}
}
}
}
zk_account = "kafka-zookeeper-service-account"
zk_secret = "kakfa-zookeeper-secret"
if sdk_utils.is_strict_mode():
service_options = sdk_utils.merge_dictionaries(
{
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
service_account_info = sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
linux_user="******",
service_account=zk_account,
service_account_secret=zk_secret)
sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
}
}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME,
service_account_info)
def zookeeper_server(configure_security):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"virtual_network_enabled": True
}
}
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
if sdk_utils.is_strict_mode():
service_options = sdk_utils.merge_dictionaries(
{
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
service_account_info = sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
linux_user="******",
service_account=zk_account,
service_account_secret=zk_secret)
sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
}
}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME,
service_account_info)
def install(package_name,
service_name,
expected_running_tasks,
additional_options={},
package_version=None,
timeout_seconds=25 * 60,
wait_for_deployment=True):
test_options = {"brokers": {"cpus": 0.5}}
sdk_install.install(package_name=package_name,
expected_running_tasks=expected_running_tasks,
service_name=service_name,
additional_options=sdk_utils.merge_dictionaries(
test_options, additional_options),
package_version=package_version,
timeout_seconds=timeout_seconds,
wait_for_deployment=wait_for_deployment)
def kafka_server_with_health_check(request, health_check_method,
service_account, kerberos):
"""
A pytest fixture that installs a kafka service.
On teardown, the service is uninstalled.
"""
service_options = {
"service": {
"name": config.SERVICE_NAME,
"health_check": {
"enabled": True,
"method": health_check_method,
"interval": 60,
"delay": 20,
"timeout": 60,
"grace-period": 30,
"max-consecutive-failures": 3,
"health-check-topic-prefix": "KafkaHealthCheck"
}
}
}
if re.search("SASL", request.param):
service_options = sdk_utils.merge_dictionaries(
{
"service": {
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": kerberos.get_realm(),
"keytab_secret": kerberos.get_keytab_path(),
}
}
}
}, service_options)
if re.search("SSL", request.param):
service_options = sdk_utils.merge_dictionaries(
{
"service": {
"service_account": service_account["name"],
"service_account_secret": service_account["secret"],
"security": {
"transport_encryption": {
"enabled": True
}
}
}
}, service_options)
if re.search("^SSL$", request.param):
service_options = sdk_utils.merge_dictionaries(
{
"service": {
"security": {
"ssl_authentication": {
"enabled": True
}
}
}
}, service_options)
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
try:
sdk_install.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
config.DEFAULT_BROKER_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
)
yield {
**{
"package_name": config.PACKAGE_NAME
}, "method": health_check_method,
"mode": request.param
}
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
def install_jmx_configured_cassandra(self_signed_trust_store: bool = True,
authentication: bool = True):
foldered_name = config.get_foldered_service_name()
test_jobs: List[Dict[str, Any]] = []
if authentication:
test_jobs = config.get_all_jobs(
node_address=config.get_foldered_node_address(), auth=True)
else:
test_jobs = config.get_all_jobs(
node_address=config.get_foldered_node_address())
# destroy/reinstall any prior leftover jobs, so that they don't touch the newly installed service:
for job in test_jobs:
sdk_jobs.install_job(job)
sdk_install.uninstall(config.PACKAGE_NAME, foldered_name)
install_jmx_secrets()
service_options = {
"service": {
"name": foldered_name,
"jmx": {
"enabled": True,
"rmi_port": 31198,
"password_file": PASSWORD_FILE,
"access_file": ACCESS_FILE,
"key_store": KEY_STORE,
"key_store_password_file": KEY_STORE_PASS,
},
}
}
if self_signed_trust_store:
service_options = sdk_utils.merge_dictionaries(
{
"service": {
"jmx": {
"add_trust_store": True,
"trust_store": TRUST_STORE,
"trust_store_password_file": TRUST_STORE_PASS,
}
}
},
service_options,
)
if authentication:
secret_path = foldered_name + "/" + config.SECRET_VALUE
create_secret(secret_value=config.SECRET_VALUE,
secret_path=secret_path)
service_options = sdk_utils.merge_dictionaries(
{
"service": {
"security": {
"authentication": {
"enabled": True,
"superuser": {
"password_secret_path": secret_path
},
},
"authorization": {
"enabled": True
},
}
}
},
service_options,
)
sdk_install.install(
config.PACKAGE_NAME,
foldered_name,
config.DEFAULT_TASK_COUNT,
additional_options=service_options,
)
