def __create_and_upload_secret(self, keytab_path: str):
"""
This method base64 encodes the keytab file and creates a secret with this encoded content so the
tasks can fetch it.
"""
log.info("Creating and uploading the keytab file %s to the secret store", keytab_path)
encoding_options = self.__encode_secret(keytab_path)
sdk_security.install_enterprise_cli()
# try to delete any preexisting secret data:
sdk_security.delete_secret(self.keytab_secret_path)
# create new secret:
cmd_list = ["security",
"secrets",
"create",
self.get_keytab_path(),
]
cmd_list.extend(encoding_options)
create_secret_cmd = " ".join(cmd_list)
log.info("Creating secret %s: %s", self.get_keytab_path(), create_secret_cmd)
rc, stdout, stderr = sdk_cmd.run_raw_cli(create_secret_cmd)
if rc != 0:
raise RuntimeError("Failed ({}) to create secret: {}\nstdout: {}\nstderr: {}".format(rc, create_secret_cmd, stdout, stderr))
log.info("Successfully uploaded a base64-encoded keytab file to the secret store")
def __create_and_upload_secret(self, keytab_path: str):
"""
This method base64 encodes the keytab file and creates a secret with this encoded content so the
tasks can fetch it.
"""
log.info(
"Creating and uploading the keytab file %s to the secret store",
keytab_path)
encoding_options = self.__encode_secret(keytab_path)
sdk_security.install_enterprise_cli()
# try to delete any preexisting secret data:
sdk_security.delete_secret(self.keytab_secret_path)
# create new secret:
cmd_list = ["security", "secrets", "create", self.get_keytab_path()]
cmd_list.extend(encoding_options)
create_secret_cmd = " ".join(cmd_list)
log.info("Creating secret %s: %s", self.get_keytab_path(),
create_secret_cmd)
sdk_cmd.run_cli(create_secret_cmd, check=True)
log.info(
"Successfully uploaded a base64-encoded keytab file to the secret store"
)
def teardown(args: dict):
log.info("Tearing down KDC")
sdk_cmd.run_cli(" ".join(["marathon", "app", "remove", "kdc"]))
sdk_cmd.run_cli(" ".join(["package", "install", "--yes", "--cli", "dcos-enterprise-cli"]))
sdk_security.delete_secret('__dcos_base64__{}'.format(args.secret_name))
log.info("KDC cluster successfully torn down")
def cleanup(self):
log.info("Removing the marathon KDC app")
sdk_marathon.destroy_app(KERBEROS_APP_ID)
log.info("Deleting temporary working directory")
self.temp_working_dir.cleanup()
#TODO: separate secrets handling into another module
log.info("Deleting keytab secret")
sdk_security.delete_secret(self.keytab_secret_path)
def teardown(args: dict):
log.info("Tearing down KDC")
sdk_cmd.run_cli(" ".join(["marathon", "app", "remove", "kdc"]))
sdk_security.install_enterprise_cli()
if args.binary_secret:
sdk_security.delete_secret(args.secret_name)
else:
sdk_security.delete_secret("__dcos_base64__{}".format(args.secret_name))
log.info("KDC cluster successfully torn down")
def teardown(args: dict):
log.info("Tearing down KDC")
sdk_cmd.run_cli(" ".join(["marathon", "app", "remove", "kdc"]))
sdk_security.install_enterprise_cli()
if args.binary_secret:
sdk_security.delete_secret(args.secret_name)
else:
sdk_security.delete_secret("__dcos_base64__{}".format(args.secret_name))
log.info("KDC cluster successfully torn down")
def cleanup(self):
sdk_security.install_enterprise_cli()
log.info("Removing the marathon KDC app")
sdk_marathon.destroy_app(self.app_definition["id"])
if self._temp_working_dir and isinstance(self._temp_working_dir, tempfile.TemporaryDirectory):
log.info("Deleting temporary working directory")
self._temp_working_dir.cleanup()
# TODO: separate secrets handling into another module
log.info("Deleting keytab secret")
sdk_security.delete_secret(self.keytab_secret_path)
def test_rpc_auth():
secret_name = "sparkauth"
sdk_security.delete_secret(secret_name)
rc, _, _ = sdk_cmd.run_raw_cli("{} --verbose secret /{}".format(utils.SPARK_PACKAGE_NAME, secret_name))
assert rc == 0, "Failed to generate Spark auth secret"
utils.run_tests(
app_url=utils.SPARK_EXAMPLES,
app_args="100",
expected_output="Pi is roughly 3",
service_name=utils.SPARK_SERVICE_NAME,
args=["--executor-auth-secret {}".format(secret_name),
"--class org.apache.spark.examples.SparkPi"])
def __create_and_upload_secret(self, keytab_path: str):
"""
This method base64 encodes the keytab file and creates a secret with this encoded content so the
tasks can fetch it.
"""
log.info(
"Creating and uploading the keytab file %s to the secret store",
keytab_path)
try:
base64_encoded_keytab_path = "{}.base64".format(keytab_path)
with open(keytab_path, "rb") as f:
keytab = f.read()
base64_encoding = base64.b64encode(keytab).decode("utf-8")
with open(base64_encoded_keytab_path, "w") as f:
f.write(base64_encoding)
log.info("Finished base64-encoding secret content (%d bytes): %s",
len(base64_encoding), base64_encoding)
except Exception as e:
raise Exception(
"Failed to base64-encode the keytab file: {}".format(repr(e)))
self.keytab_secret_path = "{}_keytab".format(DCOS_BASE64_PREFIX)
sdk_security.install_enterprise_cli()
# try to delete any preexisting secret data:
sdk_security.delete_secret(self.keytab_secret_path)
# create new secret:
create_secret_cmd = "security secrets create {keytab_secret_path} --value-file {encoded_keytab_path}".format(
keytab_secret_path=self.keytab_secret_path,
encoded_keytab_path=base64_encoded_keytab_path)
log.info("Creating secret named %s from file %s: %s",
self.keytab_secret_path, base64_encoded_keytab_path,
create_secret_cmd)
rc, stdout, stderr = sdk_cmd.run_raw_cli(create_secret_cmd)
if rc != 0:
raise RuntimeError(
"Failed ({}) to create secret: {}\nstdout: {}\nstderr: {}".
format(rc, create_secret_cmd, stdout, stderr))
log.info(
"Successfully uploaded a base64-encoded keytab file to the secret store"
)
def test_s3_secrets():
linecount_path = os.path.join(THIS_DIR, 'resources', 'linecount.txt')
s3.upload_file(linecount_path)
creds = s3.get_credentials()
def make_credential_secret(path, val):
sdk_security.delete_secret(path)
rc, stdout, stderr = sdk_cmd.run_raw_cli(
"security secrets create /{} -v {}".format(path, val))
assert rc == 0, "Failed to create secret {}, stderr: {}, stdout: {}".format(
path, stderr, stdout)
aws_access_key_path = "aws_access_key_id"
make_credential_secret(aws_access_key_path, creds.access_key)
aws_secret_key_path = "aws_secret_access_key"
make_credential_secret(aws_secret_key_path, creds.secret_key)
args = [
"--conf spark.mesos.containerizer=mesos",
"--conf spark.mesos.driver.secret.names=/{key},/{secret}".format(
key=aws_access_key_path, secret=aws_secret_key_path),
"--conf spark.mesos.driver.secret.envkeys=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY",
"--class S3Job"
]
try:
# download/read linecount.txt only
utils.run_tests(app_url=utils.dcos_test_jar_url(),
app_args="--readUrl {} --countOnly".format(
s3.s3n_url('linecount.txt')),
expected_output="Read 3 lines",
args=args)
# download/read linecount.txt, reupload as linecount-secret.txt:
utils.run_tests(app_url=utils.dcos_test_jar_url(),
app_args="--readUrl {} --writeUrl {}".format(
s3.s3n_url('linecount.txt'),
s3.s3n_url('linecount-secret.txt')),
expected_output="Read 3 lines",
args=args)
assert len(list(s3.list("linecount-secret.txt"))) > 0
finally:
sdk_security.delete_secret(aws_access_key_path)
sdk_security.delete_secret(aws_secret_key_path)
def cleanup(self) -> None:
sdk_security.install_enterprise_cli()
log.info("Removing the marathon KDC app")
sdk_marathon.destroy_app(self.app_definition["id"])
if self._temp_working_dir and isinstance(self._temp_working_dir,
tempfile.TemporaryDirectory):
log.info("Deleting temporary working directory")
self._temp_working_dir.cleanup()
sdk_security.delete_service_account(
service_account_name=KDC_SERVICE_ACCOUNT,
service_account_secret=KDC_SERVICE_ACCOUNT_SECRET,
)
# TODO: separate secrets handling into another module
log.info("Deleting keytab secret")
sdk_security.install_enterprise_cli()
sdk_security.delete_secret(self.get_keytab_path())
def make_credential_secret(path, val):
sdk_security.delete_secret(path)
rc, stdout, stderr = sdk_cmd.run_raw_cli(
"security secrets create /{} -v {}".format(path, val))
assert rc == 0, "Failed to create secret {}, stderr: {}, stdout: {}".format(
path, stderr, stdout)
def create_secret(secret_value: str, secret_path: str) -> None:
sdk_security.delete_secret(secret_path)
sdk_cmd.run_cli(
'security secrets create --value="{account}" "{secret}"'.format(
account=secret_value, secret=secret_path))
def uninstall_jmx_secrets():
sdk_security.delete_secret(KEY_STORE)
sdk_security.delete_secret(KEY_STORE_PASS)
sdk_security.delete_secret(ACCESS_FILE)
sdk_security.delete_secret(PASSWORD_FILE)
