def _create_service_accounts(service_name, security=None):
if security == DCOS_SECURITY.strict:
try:
start = time.time()
log.info("Creating service accounts for '{}'".format(service_name))
sanitized_service_name = service_name.strip("/").replace("/", "__")
sa_name = "{}-principal".format(sanitized_service_name)
sa_secret = "jenkins-{}-secret".format(sanitized_service_name)
sdk_security.create_service_account(
sa_name, sa_secret, sanitized_service_name
)
sdk_security.grant_permissions("root", "*", sa_name)
sdk_security.grant_permissions("root", SHARED_ROLE, sa_name)
end = time.time()
ACCOUNTS[service_name] = {}
ACCOUNTS[service_name]["sa_name"] = sa_name
ACCOUNTS[service_name]["sa_secret"] = sa_secret
TIMINGS["serviceaccounts"][sanitized_service_name] = end - start
except Exception as e:
log.warning(
"Error encountered while creating service account: {}".format(e)
)
raise e
def install(self) -> Dict[str, Any]:
if sdk_marathon.app_exists(self.app_definition["id"]):
if self._persist:
log.info("Found installed KDC app, reusing it")
return _get_kdc_task(self.app_definition["id"])
log.info("Found installed KDC app, destroying it first")
sdk_marathon.destroy_app(self.app_definition["id"])
# (re-)create a service account for the KDC service
sdk_security.create_service_account(
service_account_name=KDC_SERVICE_ACCOUNT,
service_account_secret=KDC_SERVICE_ACCOUNT_SECRET,
)
sdk_security._grant(
KDC_SERVICE_ACCOUNT,
"dcos:secrets:default:%252F*",
"Create any secret in the root path",
"create",
)
sdk_security._grant(
KDC_SERVICE_ACCOUNT,
"dcos:secrets:default:%252F*",
"Update any secret in the root path",
"update",
)
log.info("Installing KDC Marathon app")
sdk_marathon.install_app(self.app_definition)
log.info("KDC app installed successfully")
log.info("Waiting for KDC web API endpoint to become available")
self.__wait_for_kdc_api()
log.info("KDC web API is now available")
return _get_kdc_task(self.app_definition["id"])
def configure_package(configure_security):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
# Create service account
sdk_security.create_service_account(
service_account_name=config.SERVICE_NAME,
service_account_secret=config.SERVICE_NAME + "-secret",
)
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(
name=config.SERVICE_NAME))
sdk_install.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
6,
additional_options={
"service": {
"yaml": "tls",
"service_account": config.SERVICE_NAME,
"service_account_secret": config.SERVICE_NAME + "-secret",
# Legacy values
"principal": config.SERVICE_NAME,
"secret_name": config.SERVICE_NAME,
},
"tls": {
"discovery_task_prefix": DISCOVERY_TASK_PREFIX
},
},
)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
yield # let the test session execute
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
sdk_security.delete_service_account(
service_account_name=config.SERVICE_NAME,
service_account_secret=config.SERVICE_NAME + "-secret",
)
# Make sure that all the TLS artifacts were removed from the secrets store.
_, output, _ = sdk_cmd.run_cli(
"security secrets list {name}".format(name=config.SERVICE_NAME))
artifact_suffixes = [
"certificate",
"private-key",
"root-ca-certificate",
"keystore",
"truststore",
]
for suffix in artifact_suffixes:
assert suffix not in output
def setup_security():
LOGGER.info('Setting up strict-mode security for Spark')
sdk_security.create_service_account(service_account_name=service_account, service_account_secret=secret)
sdk_security.grant_permissions(
linux_user=user,
role_name=role,
service_account_name=service_account
)
grant_driver_permission(service_account, SPARK_APP_NAME)
grant_driver_permission(service_account, FOLDERED_SPARK_APP_NAME)
LOGGER.info('Finished setting up strict-mode security for Spark')
def configure_package(configure_security):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
# Create service account
sdk_security.create_service_account(
service_account_name=config.SERVICE_NAME,
service_account_secret=config.SERVICE_NAME)
# TODO(mh): Fine grained permissions needs to be addressed in DCOS-16475
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(
name=config.SERVICE_NAME))
sdk_install.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
6,
additional_options={
"service": {
"spec_file": "examples/tls.yml",
"service_account": config.SERVICE_NAME,
"service_account_secret": config.SERVICE_NAME,
# Legacy values
"principal": config.SERVICE_NAME,
"secret_name": config.SERVICE_NAME,
},
"tls": {
"discovery_task_prefix": DISCOVERY_TASK_PREFIX,
},
})
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
# Wait for service health check to pass
shakedown.service_healthy(config.SERVICE_NAME)
yield # let the test session execute
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
sdk_security.delete_service_account(
service_account_name=config.SERVICE_NAME,
service_account_secret=config.SERVICE_NAME)
# Make sure that all the TLS artifacts were removed from the secrets store.
output = sdk_cmd.run_cli(
'security secrets list {name}'.format(name=config.SERVICE_NAME))
artifact_suffixes = [
'certificate', 'private-key', 'root-ca-certificate', 'keystore',
'truststore'
]
for suffix in artifact_suffixes:
assert suffix not in output
def setup_security():
log.info('Setting up strict-mode security for Spark')
sdk_security.create_service_account(service_account_name=service_account, service_account_secret=secret)
for user in users:
grant_user_permissions(user, role, service_account)
for service_name in service_names:
grant_launch_task_permission(service_name)
log.info('Finished setting up strict-mode security for Spark')
def service_account(dcos_security_cli):
"""
Creates service account with `hello-world` name and yields the name.
"""
name = 'hello-world'
sdk_security.create_service_account(
service_account_name=name, secret_name=name)
# TODO(mh): Fine grained permissions needs to be addressed in DCOS-16475
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(name=name))
yield name
sdk_security.delete_service_account(
service_account_name=name, secret_name=name)
def service_account():
"""
Creates service account with `elastic` name and yields the name.
"""
name = config.SERVICE_NAME
sdk_security.create_service_account(service_account_name=name,
service_account_secret=name)
# TODO(mh): Fine grained permissions needs to be addressed in DCOS-16475
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(name=name))
yield name
sdk_security.delete_service_account(service_account_name=name,
service_account_secret=name)
def service_account():
"""
Creates service account and yields the name.
"""
name = config.SERVICE_NAME
sdk_security.create_service_account(
service_account_name=name, service_account_secret=name)
# TODO(mh): Fine grained permissions needs to be addressed in DCOS-16475
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(name=name))
yield name
sdk_security.delete_service_account(
service_account_name=name, service_account_secret=name)
def configure_package(configure_security):
try:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
# Create service account
sdk_security.create_service_account(
service_account_name=config.SERVICE_NAME, service_account_secret=config.SERVICE_NAME
)
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(name=config.SERVICE_NAME)
)
sdk_install.install(
config.PACKAGE_NAME,
config.SERVICE_NAME,
6,
additional_options={
"service": {
"yaml": "tls",
"service_account": config.SERVICE_NAME,
"service_account_secret": config.SERVICE_NAME,
# Legacy values
"principal": config.SERVICE_NAME,
"secret_name": config.SERVICE_NAME,
},
"tls": {"discovery_task_prefix": DISCOVERY_TASK_PREFIX},
},
)
sdk_plan.wait_for_completed_deployment(config.SERVICE_NAME)
yield # let the test session execute
finally:
sdk_install.uninstall(config.PACKAGE_NAME, config.SERVICE_NAME)
sdk_security.delete_service_account(
service_account_name=config.SERVICE_NAME, service_account_secret=config.SERVICE_NAME
)
# Make sure that all the TLS artifacts were removed from the secrets store.
_, output, _ = sdk_cmd.run_cli("security secrets list {name}".format(name=config.SERVICE_NAME))
artifact_suffixes = [
"certificate",
"private-key",
"root-ca-certificate",
"keystore",
"truststore",
]
for suffix in artifact_suffixes:
assert suffix not in output
def service_account():
"""
Creates service account with `hello-world` name and yields the name.
"""
# This name should be same as SERVICE_NAME as it determines scheduler DCOS_LABEL value.
name = SERVICE_NAME
sdk_security.create_service_account(service_account_name=name,
service_account_secret=name)
# TODO(mh): Fine grained permissions needs to be addressed in DCOS-16475
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(name=name))
yield name
sdk_security.delete_service_account(service_account_name=name,
service_account_secret=name)
def service_account(configure_security):
"""
Creates service account and secret and yields dict containing both.
"""
try:
name = config.SERVICE_NAME
secret = "{}-secret".format(name)
sdk_security.create_service_account(service_account_name=name,
service_account_secret=secret)
# TODO(mh): Fine grained permissions needs to be addressed in DCOS-16475
sdk_cmd.run_cli(
"security org groups add_user superusers {name}".format(name=name))
yield {"name": name, "secret": secret}
finally:
sdk_security.delete_service_account(service_account_name=name,
service_account_secret=secret)
def package_registry_session(tmpdir_factory): # _pytest.TempdirFactory
pkg_reg_repo = {}
service_uid = "pkg-reg-uid-{}".format(sdk_utils.random_string())
secret_path = "{}-secret-{}".format(service_uid, sdk_utils.random_string())
try:
sdk_security.create_service_account(service_uid, secret_path)
grant_perms_for_registry_account(service_uid)
pkg_reg_repo = install_package_registry(secret_path)
add_dcos_files_to_registry(tmpdir_factory)
yield
finally:
log.info("Teardown of package_registry_session initiated")
sdk_repository.remove_universe_repos(pkg_reg_repo)
# TODO If/when adding S3 backend, remove `Added` packages.
sdk_install.uninstall(PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME)
# No need to revoke perms, just delete the secret; the following ignores any failures.
sdk_security.delete_service_account(service_uid, secret_path)
def package_registry_session(tmpdir_factory: TempdirFactory) -> Iterator[None]:
pkg_reg_repo: Dict[str, str] = {}
service_uid = "pkg-reg-uid-{}".format(sdk_utils.random_string())
secret_path = "{}-secret-{}".format(service_uid, sdk_utils.random_string())
try:
sdk_security.create_service_account(service_uid, secret_path)
grant_perms_for_registry_account(service_uid)
pkg_reg_repo = install_package_registry(secret_path)
add_dcos_files_to_registry(tmpdir_factory)
yield
finally:
log.info("Teardown of package_registry_session initiated")
sdk_repository.remove_universe_repos(pkg_reg_repo)
# TODO If/when adding S3 backend, remove `Added` packages.
sdk_install.uninstall(PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME)
# No need to revoke perms, just delete the secret; the following ignores any failures.
sdk_security.delete_service_account(service_uid, secret_path)
def package_registry_session(tmpdir_factory): # _pytest.TempdirFactory
pkg_reg_stub = {}
pkg_reg_repo = {}
try:
# TODO Remove stub. We should install from bootstrap registry.
pkg_reg_stub = add_package_registry_stub()
service_uid = 'pkg-reg-uid-{}'.format(sdk_utils.random_string())
secret_path = '{}-secret-{}'.format(service_uid,
sdk_utils.random_string())
sdk_security.create_service_account(service_uid, secret_path)
grant_perms_for_registry_account(service_uid)
pkg_reg_repo = install_package_registry(secret_path)
add_dcos_files_to_registry(tmpdir_factory)
yield
finally:
log.info('Teardown of package_registry_session initiated')
sdk_repository.remove_universe_repos(pkg_reg_repo)
# TODO If/when adding S3 backend, remove `Added` packages.
sdk_install.uninstall(PACKAGE_REGISTRY_NAME,
PACKAGE_REGISTRY_SERVICE_NAME)
sdk_repository.remove_universe_repos(pkg_reg_stub)
# No need to revoke perms, just delete the secret.
sdk_security.delete_service_account(service_uid, secret_path)
def test_backup_and_restore_to_s3_compatible_storage() -> None:
try:
sdk_install.install(
"minio",
"minio",
expected_running_tasks=0,
package_version="0.0.13-RELEASE.2018-10-06T00-15-16Z",
wait_for_deployment=False,
)
temp_key_id = os.getenv("AWS_ACCESS_KEY_ID")
if not temp_key_id:
assert (
False
), 'AWS credentials are required for this test. Disable test with e.g. TEST_TYPES="sanity and not aws"'
temp_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY")
options = ""
if sdk_utils.is_strict_mode():
sdk_security.create_service_account(
service_account_name="marathon-lb-sa",
service_account_secret="marathon-lb/service-account-secret",
)
sdk_cmd.run_cli(
"security org users grant marathon-lb-sa dcos:service:marathon:marathon:services:/ read"
)
sdk_cmd.run_cli(
'security org users grant marathon-lb-sa dcos:service:marathon:marathon:admin:events read --description "Allows access to Marathon events"'
)
options = {
"marathon-lb": {
"secret_name": "marathon-lb/service-account-secret",
"marathon-uri": "https://marathon.mesos:8443",
}
}
sdk_install.install(
"marathon-lb",
"marathon-lb",
expected_running_tasks=0,
additional_options=options,
package_version="1.14.0",
wait_for_deployment=False,
)
host = sdk_marathon.get_scheduler_host("marathon-lb")
_, public_node_ip, _ = sdk_cmd.agent_ssh(host, "curl -s ifconfig.co")
minio_endpoint_url = "http://" + public_node_ip + ":9000"
os.environ["AWS_ACCESS_KEY_ID"] = config.MINIO_AWS_ACCESS_KEY_ID
os.environ["AWS_SECRET_ACCESS_KEY"] = config.MINIO_AWS_SECRET_ACCESS_KEY
subprocess.run(
[
"aws",
"s3",
"mb",
"s3://" + config.MINIO_BUCKET_NAME,
"--endpoint",
minio_endpoint_url,
]
)
plan_parameters = {
"AWS_ACCESS_KEY_ID": os.getenv("AWS_ACCESS_KEY_ID"),
"AWS_SECRET_ACCESS_KEY": os.getenv("AWS_SECRET_ACCESS_KEY"),
"AWS_REGION": os.getenv("AWS_REGION", "us-west-2"),
"S3_BUCKET_NAME": config.MINIO_BUCKET_NAME,
"SNAPSHOT_NAME": str(uuid.uuid1()),
"CASSANDRA_KEYSPACES": '"testspace1 testspace2"',
"S3_ENDPOINT_URL": minio_endpoint_url,
}
config.run_backup_and_restore(
config.get_foldered_service_name(),
"backup-s3",
"restore-s3",
plan_parameters,
config.get_foldered_node_address(),
)
finally:
sdk_install.uninstall("minio", "minio")
sdk_install.uninstall("marathon-lb", "marathon-lb")
os.environ["AWS_ACCESS_KEY_ID"] = temp_key_id
os.environ["AWS_SECRET_ACCESS_KEY"] = temp_secret_access_key