def test_scaling_load(
master_count,
job_count,
single_use: bool,
run_delay,
cpu_quota,
memory_quota,
work_duration,
mom,
external_volume: bool,
scenario,
min_index,
max_index,
batch_size,
enforce_quota_guarantee,
enforce_quota_limit,
create_framework: bool,
create_jobs: bool,
) -> None:
"""Launch a load test scenario. This does not verify the results
of the test, but does ensure the instances and jobs were created.
The installation is run in threads, but the job creation and
launch is then done serially after all Jenkins instances have
completed installation.
Args:
master_count: Number of Jenkins masters or instances
job_count: Number of Jobs on each Jenkins master
single_use: Mesos Single-Use Agent on (true) or off (false)
run_delay: Jobs should run every X minute(s)
cpu_quota: CPU quota (0.0 to disable)
work_duration: Time, in seconds, for generated jobs to sleep
mom: Marathon on Marathon instance name
external_volume: External volume on rexray (true) or local volume (false)
min_index: minimum index to begin jenkins suffixes at
max_index: maximum index to end jenkins suffixes at
batch_size: batch size to deploy jenkins instances in
"""
security_mode = sdk_dcos.get_security_mode()
# DELETEME@kjoshi get rid of these two after verification
# _setup_quota(SHARED_ROLE, cpu_quota, memory_quota)
if mom and cpu_quota != 0.0 and memory_quota != 0.0:
with shakedown.marathon_on_marathon(mom):
_setup_quota(SHARED_ROLE, cpu_quota, memory_quota)
# create marathon client
if mom:
_configure_admin_router(mom, SHARED_ROLE)
with shakedown.marathon_on_marathon(mom):
marathon_client = shakedown.marathon.create_client()
else:
marathon_client = shakedown.marathon.create_client()
# figure out the range of masters we want to create
if min_index == -1 or max_index == -1:
min_index = 0
max_index = master_count - 1
masters = ["jenkins/jenkins{}".format(index) for index in range(min_index, max_index)]
successful_deployments = set(masters)
# create service accounts in parallel
sdk_security.install_enterprise_cli()
security_mode = sdk_dcos.get_security_mode()
if create_framework:
log.info(
"\n\nCreating service accounts for: [{}]\n\n".format(successful_deployments)
)
service_account_creation_failures = _create_service_accounts_stage(
masters, min_index, max_index, batch_size, security_mode
)
log.info(
"\n\nService account failures: [{}]\n\n".format(
service_account_creation_failures
)
)
successful_deployments -= service_account_creation_failures
log.info(
"\n\nCreating jenkins frameworks for: [{}]\n\n".format(
successful_deployments
)
)
install_jenkins_failures = _install_jenkins_stage(
[x for x in successful_deployments],
min_index,
max_index,
batch_size,
security_mode,
marathon_client,
external_volume,
mom,
)
log.info(
"\n\nJenkins framework creation failures: [{}]\n\n".format(
install_jenkins_failures
)
)
successful_deployments -= install_jenkins_failures
if create_jobs:
log.info(
"\n\nCreating jenkins jobs for: [{}]\n\n".format(successful_deployments)
)
job_creation_failures = _create_jobs_stage(
[x for x in successful_deployments],
min_index,
max_index,
batch_size,
security_mode,
marathon_client,
external_volume,
mom,
job_count,
single_use,
run_delay,
work_duration,
scenario,
)
successful_deployments -= job_creation_failures
log.info("\n\nAll masters to deploy: [{}]\n\n".format(",".join(masters)))
log.info(
"\n\nSuccessful Jenkins deployments: [{}]\n\n".format(successful_deployments)
)
log.info(
"\n\nFailed Jenkins deployments: [{}]\n\n".format(
set(masters) - successful_deployments
)
)
log.info("Timings: {}".format(json.dumps(TIMINGS)))
def test_scaling_load(master_count, job_count, single_use: bool, run_delay,
cpu_quota, work_duration, mom, external_volume: bool,
scenario, min_index, max_index, batch_size) -> None:
"""Launch a load test scenario. This does not verify the results
of the test, but does ensure the instances and jobs were created.
The installation is run in threads, but the job creation and
launch is then done serially after all Jenkins instances have
completed installation.
Args:
master_count: Number of Jenkins masters or instances
job_count: Number of Jobs on each Jenkins master
single_use: Mesos Single-Use Agent on (true) or off (false)
run_delay: Jobs should run every X minute(s)
cpu_quota: CPU quota (0.0 to disable)
work_duration: Time, in seconds, for generated jobs to sleep
mom: Marathon on Marathon instance name
external_volume: External volume on rexray (true) or local volume (false)
min_index: minimum index to begin jenkins suffixes at
max_index: maximum index to end jenkins suffixes at
batch_size: batch size to deploy jenkins instances in
"""
security_mode = sdk_dcos.get_security_mode()
if mom and cpu_quota != 0.0:
with shakedown.marathon_on_marathon(mom):
_setup_quota(SHARED_ROLE, cpu_quota)
# create marathon client
if mom:
with shakedown.marathon_on_marathon(mom):
marathon_client = shakedown.marathon.create_client()
else:
marathon_client = shakedown.marathon.create_client()
masters = []
if min_index == -1 or max_index == -1:
masters = [
"jenkins{}".format(sdk_utils.random_string())
for _ in range(0, int(master_count))
]
else:
#max and min indexes are specified
#NOTE: using min/max will override master count
masters = [
"jenkins{}".format(index) for index in range(min_index, max_index)
]
# create service accounts in parallel
sdk_security.install_enterprise_cli()
service_account_threads = _spawn_threads(masters,
_create_service_accounts,
security=security_mode)
thread_failures = _wait_and_get_failures(service_account_threads,
timeout=SERVICE_ACCOUNT_TIMEOUT)
# launch Jenkins services
current = 0
end = max_index - min_index
while current + batch_size <= end:
log.info(
"Re-authenticating current batch load of jenkins{} - jenkins{} "
"to prevent auth-timeouts on scale cluster.".format(
current, current + batch_size))
dcos_login.login_session()
batched_masters = masters[current:current + batch_size]
install_threads = _spawn_threads(batched_masters,
_install_jenkins,
event='deployments',
client=marathon_client,
external_volume=external_volume,
security=security_mode,
daemon=True,
mom=mom)
thread_failures = _wait_and_get_failures(install_threads,
timeout=DEPLOY_TIMEOUT)
thread_names = [x.name for x in thread_failures]
# the rest of the commands require a running Jenkins instance
deployed_masters = [
x for x in batched_masters if x not in thread_names
]
job_threads = _spawn_threads(deployed_masters,
_create_jobs,
jobs=job_count,
single=single_use,
delay=run_delay,
duration=work_duration,
scenario=scenario)
_wait_on_threads(job_threads, JOB_RUN_TIMEOUT)
r = json.dumps(TIMINGS)
print(r)
current = current + batch_size
def test_scaling_load(master_count,
job_count,
single_use: bool,
run_delay,
cpu_quota,
work_duration,
mom,
external_volume: bool,
scenario) -> None:
"""Launch a load test scenario. This does not verify the results
of the test, but does ensure the instances and jobs were created.
The installation is run in threads, but the job creation and
launch is then done serially after all Jenkins instances have
completed installation.
Args:
master_count: Number of Jenkins masters or instances
job_count: Number of Jobs on each Jenkins master
single_use: Mesos Single-Use Agent on (true) or off (false)
run_delay: Jobs should run every X minute(s)
cpu_quota: CPU quota (0.0 to disable)
work_duration: Time, in seconds, for generated jobs to sleep
mom: Marathon on Marathon instance name
external_volume: External volume on rexray (true) or local volume (false)
"""
security_mode = sdk_dcos.get_security_mode()
if mom and cpu_quota != 0.0:
with shakedown.marathon_on_marathon(mom):
_setup_quota(SHARED_ROLE, cpu_quota)
# create marathon client
if mom:
with shakedown.marathon_on_marathon(mom):
marathon_client = shakedown.marathon.create_client()
else:
marathon_client = shakedown.marathon.create_client()
masters = ["jenkins{}".format(sdk_utils.random_string()) for _ in
range(0, int(master_count))]
# create service accounts in parallel
service_account_threads = _spawn_threads(masters,
_create_service_accounts,
security=security_mode)
thread_failures = _wait_and_get_failures(service_account_threads,
timeout=SERVICE_ACCOUNT_TIMEOUT)
# launch Jenkins services
install_threads = _spawn_threads(masters,
_install_jenkins,
event='deployments',
client=marathon_client,
external_volume=external_volume,
security=security_mode,
daemon=True,
mom=mom)
thread_failures = _wait_and_get_failures(install_threads,
timeout=DEPLOY_TIMEOUT)
thread_names = [x.name for x in thread_failures]
# the rest of the commands require a running Jenkins instance
deployed_masters = [x for x in masters if x not in thread_names]
job_threads = _spawn_threads(deployed_masters,
_create_jobs,
jobs=job_count,
single=single_use,
delay=run_delay,
duration=work_duration,
scenario=scenario)
_wait_on_threads(job_threads, JOB_RUN_TIMEOUT)
r = json.dumps(TIMINGS)
print(r)