def __create_and_upload_secret(self, keytab_path: str):
"""
This method base64 encodes the keytab file and creates a secret with this encoded content so the
tasks can fetch it.
"""
log.info("Creating and uploading the keytab file %s to the secret store", keytab_path)
encoding_options = self.__encode_secret(keytab_path)
sdk_security.install_enterprise_cli()
# try to delete any preexisting secret data:
sdk_security.delete_secret(self.keytab_secret_path)
# create new secret:
cmd_list = ["security",
"secrets",
"create",
self.get_keytab_path(),
]
cmd_list.extend(encoding_options)
create_secret_cmd = " ".join(cmd_list)
log.info("Creating secret %s: %s", self.get_keytab_path(), create_secret_cmd)
rc, stdout, stderr = sdk_cmd.run_raw_cli(create_secret_cmd)
if rc != 0:
raise RuntimeError("Failed ({}) to create secret: {}\nstdout: {}\nstderr: {}".format(rc, create_secret_cmd, stdout, stderr))
log.info("Successfully uploaded a base64-encoded keytab file to the secret store")
def test_scaling_cleanup(service_name, scenario, service_count, min_index,
max_index) -> None:
""" Cleanup of installed hello-world service for specified scenario
Args:
service_name: name of the service to uninstall
scenario: yaml scenario helloworld installed with (normal, crashloop)
service_count: number of helloworld services to uninstall
min_index: minimum index to begin suffixes at
max_index: maximum index to end suffixes at
"""
scale_service_name = "{}-{}".format(service_name, scenario)
if min_index == -1 or max_index == -1:
scale_cleanup_list = [
"{}-{}".format(scale_service_name, index)
for index in range(0, int(service_count))
]
else:
scale_cleanup_list = [
"{}-{}".format(scale_service_name, index)
for index in range(min_index, max_index)
]
sdk_security.install_enterprise_cli()
cleanup_threads = spawn_threads(scale_cleanup_list, _uninstall_service)
# Launch jobs.
wait_and_get_failures(cleanup_threads, timeout=JOB_RUN_TIMEOUT)
def __create_and_upload_secret(self, keytab_path: str):
"""
This method base64 encodes the keytab file and creates a secret with this encoded content so the
tasks can fetch it.
"""
log.info(
"Creating and uploading the keytab file %s to the secret store",
keytab_path)
encoding_options = self.__encode_secret(keytab_path)
sdk_security.install_enterprise_cli()
# try to delete any preexisting secret data:
sdk_security.delete_secret(self.keytab_secret_path)
# create new secret:
cmd_list = ["security", "secrets", "create", self.get_keytab_path()]
cmd_list.extend(encoding_options)
create_secret_cmd = " ".join(cmd_list)
log.info("Creating secret %s: %s", self.get_keytab_path(),
create_secret_cmd)
sdk_cmd.run_cli(create_secret_cmd, check=True)
log.info(
"Successfully uploaded a base64-encoded keytab file to the secret store"
)
def install_jmx_secrets():
uninstall_jmx_secrets()
test_run_id = random_string()
create_keystore_cmd = [
"keytool",
"-genkey",
"-alias",
"self-signed-cert",
"-keyalg",
"rsa",
"-dname",
"CN=myhost.example.com,O=Example Company,C=US",
"-keystore",
"/tmp/{}-self-signed-keystore.ks".format(test_run_id),
"-storepass",
"deleteme",
"-keypass",
"deleteme",
"-storetype",
"jks",
]
subprocess.check_output(create_keystore_cmd)
keystore_list_cmd = [
"keytool",
"-list",
"-v",
"-keystore",
"/tmp/{}-self-signed-keystore.ks".format(test_run_id),
"-storepass",
"deleteme",
]
subprocess.check_output(keystore_list_cmd)
write_to_file("deleteme", "/tmp/{}-keystorepass".format(test_run_id))
write_to_file("admin adminpassword",
"/tmp/{}-passwordfile".format(test_run_id))
write_to_file("admin readwrite", "/tmp/{}-access".format(test_run_id))
sdk_security.install_enterprise_cli(False)
sdk_cmd.run_cli(
"security secrets create -f /tmp/{}-self-signed-keystore.ks {}".format(
test_run_id, KEY_STORE))
sdk_cmd.run_cli(
"security secrets create -f /tmp/{}-passwordfile {}".format(
test_run_id, PASSWORD_FILE))
sdk_cmd.run_cli(
"security secrets create -f /tmp/{}-keystorepass {}".format(
test_run_id, KEY_STORE_PASS))
sdk_cmd.run_cli("security secrets create -f /tmp/{}-access {}".format(
test_run_id, ACCESS_FILE))
def teardown(args: dict):
log.info("Tearing down KDC")
sdk_cmd.run_cli(" ".join(["marathon", "app", "remove", "kdc"]))
sdk_security.install_enterprise_cli()
if args.binary_secret:
sdk_security.delete_secret(args.secret_name)
else:
sdk_security.delete_secret("__dcos_base64__{}".format(args.secret_name))
log.info("KDC cluster successfully torn down")
def teardown(args: dict):
log.info("Tearing down KDC")
sdk_cmd.run_cli(" ".join(["marathon", "app", "remove", "kdc"]))
sdk_security.install_enterprise_cli()
if args.binary_secret:
sdk_security.delete_secret(args.secret_name)
else:
sdk_security.delete_secret("__dcos_base64__{}".format(args.secret_name))
log.info("KDC cluster successfully torn down")
def cleanup(self):
sdk_security.install_enterprise_cli()
log.info("Removing the marathon KDC app")
sdk_marathon.destroy_app(self.app_definition["id"])
if self._temp_working_dir and isinstance(self._temp_working_dir, tempfile.TemporaryDirectory):
log.info("Deleting temporary working directory")
self._temp_working_dir.cleanup()
# TODO: separate secrets handling into another module
log.info("Deleting keytab secret")
sdk_security.delete_secret(self.keytab_secret_path)
def spark_security_session(
users=[SPARK_USER],
service_names=[SPARK_SERVICE_NAME, FOLDERED_SPARK_SERVICE_NAME]):
'''
Spark strict mode setup is slightly different from dcos-commons, so can't use sdk_security::security_session.
Differences:
(1) the role is "*", (2) the driver itself is a framework and needs permission to execute tasks.
'''
role = '*'
service_account = SPARK_SERVICE_ACCOUNT
secret = SPARK_SERVICE_ACCOUNT_SECRET
def setup_security():
log.info('Setting up strict-mode security for Spark')
sdk_security.create_service_account(
service_account_name=service_account,
service_account_secret=secret)
for user in users:
grant_user_permissions(user, role, service_account)
for service_name in service_names:
grant_launch_task_permission(service_name)
log.info('Finished setting up strict-mode security for Spark')
def cleanup_security():
log.info('Cleaning up strict-mode security for Spark')
for user in users:
revoke_user_permissions(user, role, service_account)
# TODO: improve security setup/teardown to make it more fine-grained (allow different service names/accts/users)
# tracking issue: https://jira.mesosphere.com/browse/DCOS-50933
sdk_security.delete_service_account(service_account, secret)
log.info('Finished cleaning up strict-mode security for Spark')
try:
if not sdk_utils.is_open_dcos():
sdk_security.install_enterprise_cli()
if sdk_utils.is_strict_mode():
setup_security()
yield
finally:
if sdk_utils.is_strict_mode():
cleanup_security()
def __create_and_upload_secret(self, keytab_path: str):
"""
This method base64 encodes the keytab file and creates a secret with this encoded content so the
tasks can fetch it.
"""
log.info(
"Creating and uploading the keytab file %s to the secret store",
keytab_path)
try:
base64_encoded_keytab_path = "{}.base64".format(keytab_path)
with open(keytab_path, "rb") as f:
keytab = f.read()
base64_encoding = base64.b64encode(keytab).decode("utf-8")
with open(base64_encoded_keytab_path, "w") as f:
f.write(base64_encoding)
log.info("Finished base64-encoding secret content (%d bytes): %s",
len(base64_encoding), base64_encoding)
except Exception as e:
raise Exception(
"Failed to base64-encode the keytab file: {}".format(repr(e)))
self.keytab_secret_path = "{}_keytab".format(DCOS_BASE64_PREFIX)
sdk_security.install_enterprise_cli()
# try to delete any preexisting secret data:
sdk_security.delete_secret(self.keytab_secret_path)
# create new secret:
create_secret_cmd = "security secrets create {keytab_secret_path} --value-file {encoded_keytab_path}".format(
keytab_secret_path=self.keytab_secret_path,
encoded_keytab_path=base64_encoded_keytab_path)
log.info("Creating secret named %s from file %s: %s",
self.keytab_secret_path, base64_encoded_keytab_path,
create_secret_cmd)
rc, stdout, stderr = sdk_cmd.run_raw_cli(create_secret_cmd)
if rc != 0:
raise RuntimeError(
"Failed ({}) to create secret: {}\nstdout: {}\nstderr: {}".
format(rc, create_secret_cmd, stdout, stderr))
log.info(
"Successfully uploaded a base64-encoded keytab file to the secret store"
)
def test_scaling_load(service_name, scenario, service_count, min_index,
max_index, batch_size, package_version) -> None:
"""Launch a load test scenario in parallel if needed.
This does not verify the results of the test, but does
ensure the instances were created.
Args:
service_name: name of the service to install as
scenario: yaml scenario to run helloworld with (normal, crashloop) are added for this case
service_count: number of helloworld services to install
min_index: minimum index to begin suffixes at
max_index: maximum index to end suffixes at
batch_size: batch size to deploy instances in
package_version: version of hello-world to deploy, defaults to catalog if unspecified.
"""
scale_service_name = "{}-{}".format(service_name, scenario)
deployment_list = []
if min_index == -1 or max_index == -1:
deployment_list = [
"{}-{}".format(scale_service_name, index)
for index in range(0, int(service_count))
]
min_index = 0
max_index = service_count
else:
deployment_list = [
"{}-{}".format(scale_service_name, index)
for index in range(min_index, max_index)
]
sdk_security.install_enterprise_cli()
current = 0
end = max_index - min_index
for current in range(0, end, batch_size):
# Get current batch of schedulers to deploy.
batched_deployment_list = deployment_list[current:current + batch_size]
# Create threads with correct arguments.
deployment_threads = spawn_threads(
batched_deployment_list,
_launch_load,
scenario=scenario,
package_version=package_version,
)
# Launch jobs.
wait_and_get_failures(deployment_threads, timeout=JOB_RUN_TIMEOUT)
def cleanup(self) -> None:
sdk_security.install_enterprise_cli()
log.info("Removing the marathon KDC app")
sdk_marathon.destroy_app(self.app_definition["id"])
if self._temp_working_dir and isinstance(self._temp_working_dir,
tempfile.TemporaryDirectory):
log.info("Deleting temporary working directory")
self._temp_working_dir.cleanup()
sdk_security.delete_service_account(
service_account_name=KDC_SERVICE_ACCOUNT,
service_account_secret=KDC_SERVICE_ACCOUNT_SECRET,
)
# TODO: separate secrets handling into another module
log.info("Deleting keytab secret")
sdk_security.install_enterprise_cli()
sdk_security.delete_secret(self.get_keytab_path())
def test_scaling_load(service_name,
scenario,
service_count,
min_index,
max_index,
batch_size) -> None:
"""Launch a load test scenario in parallel if needed.
This does not verify the results of the test, but does
ensure the instances were created.
Args:
service_name: name of the service to install as
scenario: yaml scenario to run helloworld with (normal, crashloop) are added for this case
service_count: number of helloworld services to install
min_index: minimum index to begin suffixes at
max_index: maximum index to end suffixes at
batch_size: batch size to deploy instances in
"""
scale_service_name = "{}-{}".format(service_name, scenario)
deployment_list = []
if min_index == -1 or max_index == -1:
deployment_list = ["{}-{}".format(scale_service_name, index) for index in range(0, int(service_count))]
min_index = 0
max_index = service_count
else:
deployment_list = ["{}-{}".format(scale_service_name, index) for index in range(min_index, max_index)]
sdk_security.install_enterprise_cli()
current = 0
end = max_index - min_index
for current in range(0, end, batch_size):
# Get current batch of schedulers to deploy.
batched_deployment_list = deployment_list[current:current + batch_size]
# Create threads with correct arguments.
deployment_threads = spawn_threads(batched_deployment_list,
_launch_load,
scenario=scenario)
# Launch jobs.
wait_and_get_failures(deployment_threads, timeout=JOB_RUN_TIMEOUT)
def test_scaling_cleanup(service_name, scenario, service_count, min_index, max_index) -> None:
""" Cleanup of installed hello-world service for specified scenario
Args:
service_name: name of the service to uninstall
scenario: yaml scenario helloworld installed with (normal, crashloop)
service_count: number of helloworld services to uninstall
min_index: minimum index to begin suffixes at
max_index: maximum index to end suffixes at
"""
scale_service_name = "{}-{}".format(service_name, scenario)
if min_index == -1 or max_index == -1:
scale_cleanup_list = ["{}-{}".format(scale_service_name, index) for index in range(0, int(service_count))]
else:
scale_cleanup_list = ["{}-{}".format(scale_service_name, index) for index in range(min_index, max_index)]
sdk_security.install_enterprise_cli()
cleanup_threads = spawn_threads(scale_cleanup_list,
_uninstall_service)
# Launch jobs.
wait_and_get_failures(cleanup_threads, timeout=JOB_RUN_TIMEOUT)
def test_scaling_load(master_count, job_count, single_use: bool, run_delay,
cpu_quota, work_duration, mom, external_volume: bool,
scenario, min_index, max_index, batch_size) -> None:
"""Launch a load test scenario. This does not verify the results
of the test, but does ensure the instances and jobs were created.
The installation is run in threads, but the job creation and
launch is then done serially after all Jenkins instances have
completed installation.
Args:
master_count: Number of Jenkins masters or instances
job_count: Number of Jobs on each Jenkins master
single_use: Mesos Single-Use Agent on (true) or off (false)
run_delay: Jobs should run every X minute(s)
cpu_quota: CPU quota (0.0 to disable)
work_duration: Time, in seconds, for generated jobs to sleep
mom: Marathon on Marathon instance name
external_volume: External volume on rexray (true) or local volume (false)
min_index: minimum index to begin jenkins suffixes at
max_index: maximum index to end jenkins suffixes at
batch_size: batch size to deploy jenkins instances in
"""
security_mode = sdk_dcos.get_security_mode()
if mom and cpu_quota != 0.0:
with shakedown.marathon_on_marathon(mom):
_setup_quota(SHARED_ROLE, cpu_quota)
# create marathon client
if mom:
with shakedown.marathon_on_marathon(mom):
marathon_client = shakedown.marathon.create_client()
else:
marathon_client = shakedown.marathon.create_client()
masters = []
if min_index == -1 or max_index == -1:
masters = [
"jenkins{}".format(sdk_utils.random_string())
for _ in range(0, int(master_count))
]
else:
#max and min indexes are specified
#NOTE: using min/max will override master count
masters = [
"jenkins{}".format(index) for index in range(min_index, max_index)
]
# create service accounts in parallel
sdk_security.install_enterprise_cli()
service_account_threads = _spawn_threads(masters,
_create_service_accounts,
security=security_mode)
thread_failures = _wait_and_get_failures(service_account_threads,
timeout=SERVICE_ACCOUNT_TIMEOUT)
# launch Jenkins services
current = 0
end = max_index - min_index
while current + batch_size <= end:
log.info(
"Re-authenticating current batch load of jenkins{} - jenkins{} "
"to prevent auth-timeouts on scale cluster.".format(
current, current + batch_size))
dcos_login.login_session()
batched_masters = masters[current:current + batch_size]
install_threads = _spawn_threads(batched_masters,
_install_jenkins,
event='deployments',
client=marathon_client,
external_volume=external_volume,
security=security_mode,
daemon=True,
mom=mom)
thread_failures = _wait_and_get_failures(install_threads,
timeout=DEPLOY_TIMEOUT)
thread_names = [x.name for x in thread_failures]
# the rest of the commands require a running Jenkins instance
deployed_masters = [
x for x in batched_masters if x not in thread_names
]
job_threads = _spawn_threads(deployed_masters,
_create_jobs,
jobs=job_count,
single=single_use,
delay=run_delay,
duration=work_duration,
scenario=scenario)
_wait_on_threads(job_threads, JOB_RUN_TIMEOUT)
r = json.dumps(TIMINGS)
print(r)
current = current + batch_size
def test_scaling_load(
master_count,
job_count,
single_use: bool,
run_delay,
cpu_quota,
memory_quota,
work_duration,
mom,
external_volume: bool,
scenario,
min_index,
max_index,
batch_size,
enforce_quota_guarantee,
enforce_quota_limit,
create_framework: bool,
create_jobs: bool,
) -> None:
"""Launch a load test scenario. This does not verify the results
of the test, but does ensure the instances and jobs were created.
The installation is run in threads, but the job creation and
launch is then done serially after all Jenkins instances have
completed installation.
Args:
master_count: Number of Jenkins masters or instances
job_count: Number of Jobs on each Jenkins master
single_use: Mesos Single-Use Agent on (true) or off (false)
run_delay: Jobs should run every X minute(s)
cpu_quota: CPU quota (0.0 to disable)
work_duration: Time, in seconds, for generated jobs to sleep
mom: Marathon on Marathon instance name
external_volume: External volume on rexray (true) or local volume (false)
min_index: minimum index to begin jenkins suffixes at
max_index: maximum index to end jenkins suffixes at
batch_size: batch size to deploy jenkins instances in
"""
security_mode = sdk_dcos.get_security_mode()
# DELETEME@kjoshi get rid of these two after verification
# _setup_quota(SHARED_ROLE, cpu_quota, memory_quota)
if mom and cpu_quota != 0.0 and memory_quota != 0.0:
with shakedown.marathon_on_marathon(mom):
_setup_quota(SHARED_ROLE, cpu_quota, memory_quota)
# create marathon client
if mom:
_configure_admin_router(mom, SHARED_ROLE)
with shakedown.marathon_on_marathon(mom):
marathon_client = shakedown.marathon.create_client()
else:
marathon_client = shakedown.marathon.create_client()
# figure out the range of masters we want to create
if min_index == -1 or max_index == -1:
min_index = 0
max_index = master_count - 1
masters = ["jenkins/jenkins{}".format(index) for index in range(min_index, max_index)]
successful_deployments = set(masters)
# create service accounts in parallel
sdk_security.install_enterprise_cli()
security_mode = sdk_dcos.get_security_mode()
if create_framework:
log.info(
"\n\nCreating service accounts for: [{}]\n\n".format(successful_deployments)
)
service_account_creation_failures = _create_service_accounts_stage(
masters, min_index, max_index, batch_size, security_mode
)
log.info(
"\n\nService account failures: [{}]\n\n".format(
service_account_creation_failures
)
)
successful_deployments -= service_account_creation_failures
log.info(
"\n\nCreating jenkins frameworks for: [{}]\n\n".format(
successful_deployments
)
)
install_jenkins_failures = _install_jenkins_stage(
[x for x in successful_deployments],
min_index,
max_index,
batch_size,
security_mode,
marathon_client,
external_volume,
mom,
)
log.info(
"\n\nJenkins framework creation failures: [{}]\n\n".format(
install_jenkins_failures
)
)
successful_deployments -= install_jenkins_failures
if create_jobs:
log.info(
"\n\nCreating jenkins jobs for: [{}]\n\n".format(successful_deployments)
)
job_creation_failures = _create_jobs_stage(
[x for x in successful_deployments],
min_index,
max_index,
batch_size,
security_mode,
marathon_client,
external_volume,
mom,
job_count,
single_use,
run_delay,
work_duration,
scenario,
)
successful_deployments -= job_creation_failures
log.info("\n\nAll masters to deploy: [{}]\n\n".format(",".join(masters)))
log.info(
"\n\nSuccessful Jenkins deployments: [{}]\n\n".format(successful_deployments)
)
log.info(
"\n\nFailed Jenkins deployments: [{}]\n\n".format(
set(masters) - successful_deployments
)
)
log.info("Timings: {}".format(json.dumps(TIMINGS)))
