def install(self, kerberos: sdk_auth.KerberosEnvironment=None) -> dict:
options = {
"id": self.id,
"mem": 512,
"container": {
"type": "MESOS",
"docker": {
"image": "elezar/kafka-client:deca3d0",
"forcePullImage": True
}
},
"networks": [
{
"mode": "host"
}
],
"env": {
"JVM_MaxHeapSize": "512",
"KAFKA_CLIENT_MODE": "test",
"KAFKA_TOPIC": "securetest"
}
}
if kerberos is not None:
self._is_kerberos = True
options = sdk_install.merge_dictionaries(options, self._get_kerberos_options(kerberos))
sdk_marathon.install_app(options)
return options
def configure_zookeeper(configure_security, install_zookeeper_stub):
service_options = {
"service": {
"virtual_network_enabled": True
}
}
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
if sdk_utils.is_strict_mode():
service_options = sdk_install.merge_dictionaries({
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
try:
sdk_install.uninstall(ZK_PACKAGE, ZK_SERVICE_NAME)
sdk_security.setup_security(ZK_SERVICE_NAME, zk_account, zk_secret)
sdk_install.install(
ZK_PACKAGE,
ZK_SERVICE_NAME,
6,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield
finally:
sdk_install.uninstall(ZK_PACKAGE, ZK_SERVICE_NAME)
def get_spark_options(service_name, additional_options):
options = {"service": {"user": SPARK_USER, "name": service_name}}
if SPARK_DOCKER_USER is not None:
options["service"]["docker_user"] = SPARK_DOCKER_USER
if sdk_utils.is_strict_mode():
# At the moment, we do this by hand because Spark doesn't quite line up with other services
# with these options, and sdk_install assumes that we're following those conventions
# Specifically, Spark's config.json lacks: service.principal, service.secret_name
options["service"]["service_account"] = SPARK_SERVICE_ACCOUNT
options["service"][
"service_account_secret"] = SPARK_SERVICE_ACCOUNT_SECRET
print(sdk_install.merge_dictionaries(options, additional_options))
return sdk_install.merge_dictionaries(options, additional_options)
def zookeeper_server(kerberos):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": kerberos.get_realm(),
"keytab_secret": kerberos.get_keytab_path(),
}
}
}
}
zk_account = "kafka-zookeeper-service-account"
zk_secret = "kakfa-zookeeper-secret"
if sdk_utils.is_strict_mode():
service_options = sdk_install.merge_dictionaries(
{
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
service_account_info = sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
linux_user="******",
service_account=zk_account,
service_account_secret=zk_secret)
sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
}
}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME,
service_account_info)
def merge_service_options(options: typing.List[typing.Dict]) -> typing.Dict:
"""
Merge the specified service options with the options later in the list taking precedence.
"""
result = {}
for o in options:
result = sdk_install.merge_dictionaries(result, o)
return result
def zookeeper_server(kerberos):
service_kerberos_options = {
"service": {
"name": "kafka-zookeeper",
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": sdk_auth.REALM,
"keytab_secret": kerberos.get_keytab_path(),
}
}
}
}
zk_account = "kafka-zookeeper-service-account"
zk_secret = "kakfa-zookeeper-secret"
if sdk_utils.is_strict_mode():
service_kerberos_options = sdk_install.merge_dictionaries(
{
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_kerberos_options)
try:
sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper")
sdk_security.setup_security("kafka-zookeeper", zk_account, zk_secret)
sdk_install.install("beta-kafka-zookeeper",
"kafka-zookeeper",
6,
additional_options=service_kerberos_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {
**service_kerberos_options,
**{
"package_name": "beta-kafka-zookeeper"
}
}
finally:
sdk_install.uninstall("beta-kafka-zookeeper", "kafka-zookeeper")
def get_strict_mode_options(service_account_info: typing.Dict) -> typing.Dict:
options = {}
if "linux_user" in service_account_info:
user_options = {
"service": {
"user": service_account_info["linux_user"]
}
}
options = sdk_install.merge_dictionaries(options, user_options)
if sdk_utils.is_strict_mode():
service_account_options = {
'service': {
'service_account': service_account_info["name"],
'service_account_secret': service_account_info["secret"],
}
}
options = sdk_install.merge_dictionaries(options, service_account_options)
return options
def zookeeper_server(configure_security):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"virtual_network_enabled": True
}
}
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
if sdk_utils.is_strict_mode():
service_options = sdk_install.merge_dictionaries(
{
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
service_account_info = sdk_security.setup_security(
config.ZOOKEEPER_SERVICE_NAME,
linux_user="******",
service_account=zk_account,
service_account_secret=zk_secret)
sdk_install.install(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {
**service_options,
**{
"package_name": config.ZOOKEEPER_PACKAGE_NAME
}
}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME)
sdk_security.cleanup_security(config.ZOOKEEPER_SERVICE_NAME,
service_account_info)
def install(package_name,
service_name,
expected_running_tasks,
additional_options={},
package_version=None,
timeout_seconds=25 * 60,
wait_for_deployment=True):
test_options = {"brokers": {"cpus": 0.5}}
sdk_install.install(package_name=package_name,
expected_running_tasks=expected_running_tasks,
service_name=service_name,
additional_options=sdk_install.merge_dictionaries(
test_options, additional_options),
package_version=package_version,
timeout_seconds=timeout_seconds,
wait_for_deployment=wait_for_deployment)
def zookeeper_server(kerberos):
service_kerberos_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"security": {
"kerberos": {
"enabled": True,
"kdc": {
"hostname": kerberos.get_host(),
"port": int(kerberos.get_port())
},
"realm": sdk_auth.REALM,
"keytab_secret": kerberos.get_keytab_path(),
}
}
}
}
zk_account = "kafka-zookeeper-service-account"
zk_secret = "kakfa-zookeeper-secret"
if sdk_utils.is_strict_mode():
service_kerberos_options = sdk_install.merge_dictionaries({
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_kerberos_options)
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
sdk_security.setup_security(config.ZOOKEEPER_SERVICE_NAME, zk_account, zk_secret)
sdk_install.install(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_kerberos_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {**service_kerberos_options, **{"package_name": config.ZOOKEEPER_PACKAGE_NAME}}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
def spark_dispatcher(configure_security_spark, configure_universe,
use_ucr_containerizer):
utils.teardown_spark(service_name=CNI_DISPATCHER_SERVICE_NAME,
zk=CNI_DISPATCHER_ZK)
options = {"service": {"UCR_containerizer": use_ucr_containerizer}}
try:
merged_options = sdk_install.merge_dictionaries(
CNI_SERVICE_OPTIONS, options)
sdk_install.install(utils.SPARK_PACKAGE_NAME,
CNI_DISPATCHER_SERVICE_NAME,
0,
additional_options=utils.get_spark_options(
CNI_DISPATCHER_SERVICE_NAME, merged_options),
wait_for_deployment=False)
yield
finally:
utils.teardown_spark(service_name=CNI_DISPATCHER_SERVICE_NAME,
zk=CNI_DISPATCHER_ZK)
def install(
package_name,
service_name,
expected_running_tasks,
additional_options={},
package_version=None,
timeout_seconds=25*60,
wait_for_deployment=True):
test_options = {
"brokers": {
"cpus": 0.5
}
}
sdk_install.install(package_name=package_name,
expected_running_tasks=expected_running_tasks,
service_name=service_name,
additional_options=sdk_install.merge_dictionaries(test_options, additional_options),
package_version=package_version,
timeout_seconds=timeout_seconds,
wait_for_deployment=wait_for_deployment)
def zookeeper_server(configure_security):
service_options = {
"service": {
"name": config.ZOOKEEPER_SERVICE_NAME,
"virtual_network_enabled": True
}
}
zk_account = "test-zookeeper-service-account"
zk_secret = "test-zookeeper-secret"
try:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
if sdk_utils.is_strict_mode():
service_options = sdk_install.merge_dictionaries({
'service': {
'service_account': zk_account,
'service_account_secret': zk_secret,
}
}, service_options)
sdk_security.setup_security(config.ZOOKEEPER_SERVICE_NAME, zk_account, zk_secret)
sdk_install.install(
config.ZOOKEEPER_PACKAGE_NAME,
config.ZOOKEEPER_SERVICE_NAME,
config.ZOOKEEPER_TASK_COUNT,
additional_options=service_options,
timeout_seconds=30 * 60,
insert_strict_options=False)
yield {**service_options, **{"package_name": config.ZOOKEEPER_PACKAGE_NAME}}
finally:
sdk_install.uninstall(config.ZOOKEEPER_PACKAGE_NAME, config.ZOOKEEPER_SERVICE_NAME)
if sdk_utils.is_strict_mode():
sdk_security.delete_service_account(
service_account_name=zk_account, service_account_secret=zk_secret)
