def build_dcos_file_from_universe_definition(
package: Dict, dcos_files_path: str, tmpdir_factory # _pytest.TempdirFactory
) -> Tuple[str, str, str]:
"""
Build the .dcos file if its not already present in the given directory.
Returns a Tuple containing (path of .dcos file, name, and version)
"""
# TODO Ideally we should `migrate` and then `build`.
name = package["name"]
version = package["version"]
target = os.path.join(dcos_files_path, "{}-{}.dcos".format(name, version))
if os.path.isfile(target):
log.info("Skipping build, using cached file : {}".format(target))
else:
del package["releaseVersion"]
del package["selected"]
package_json_file = tmpdir_factory.mktemp(sdk_utils.random_string()).join(
sdk_utils.random_string()
)
package_json_file.write(json.dumps(package))
rc, _, _ = sdk_cmd.run_cli(
" ".join(
[
"registry",
"build",
"--build-definition-file={}".format(str(package_json_file)),
"--output-directory={}".format(dcos_files_path),
"--json",
]
)
)
assert rc == 0
assert os.path.isfile(target), "No valid .dcos file is built"
return target, name, version
def build_dcos_file_from_universe_definition(
package: Dict, dcos_files_path: str, tmpdir_factory: TempdirFactory,
) -> Tuple[str, str, str]:
"""
Build the .dcos file if its not already present in the given directory.
Returns a Tuple containing (path of .dcos file, name, and version)
"""
# TODO Ideally we should `migrate` and then `build`.
name = package["name"]
version = package["version"]
target = os.path.join(dcos_files_path, "{}-{}.dcos".format(name, version))
if os.path.isfile(target):
log.info("Skipping build, using cached file : {}".format(target))
else:
del package["releaseVersion"]
del package["selected"]
package_json_file = tmpdir_factory.mktemp(sdk_utils.random_string()).join(
sdk_utils.random_string()
)
package_json_file.write(json.dumps(package))
rc, _, _ = sdk_cmd.run_cli(
" ".join(
[
"registry",
"build",
"--build-definition-file={}".format(str(package_json_file)),
"--output-directory={}".format(dcos_files_path),
"--json",
]
)
)
assert rc == 0
assert os.path.isfile(target), "No valid .dcos file is built"
return target, name, version
def add_dcos_files_to_registry(tmpdir_factory) -> None: # _pytest.TempdirFactory
# Use DCOS_FILES_PATH if its set to a valid path OR use pytest's tmpdir.
dcos_files_path = os.environ.get("DCOS_FILES_PATH", "")
valid_path_set = os.path.isdir(dcos_files_path)
if valid_path_set and not os.access(dcos_files_path, os.W_OK):
log.warning("{} is not writable.".format(dcos_files_path))
valid_path_set = False
if not valid_path_set:
dcos_files_path = str(tmpdir_factory.mktemp(sdk_utils.random_string()))
stub_universe_urls = sdk_repository.get_repos()
log.info(
"Using {} to build .dcos files (if not exists) from {}".format(
dcos_files_path, stub_universe_urls
)
)
dcos_files_list = build_dcos_files_from_stubs(
stub_universe_urls, dcos_files_path, tmpdir_factory
)
log.info("Bundled .dcos files : {}".format(dcos_files_list))
@retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000)
def wait_for_added_registry(name, version):
code, stdout, stderr = sdk_cmd.run_cli(
"registry describe --package-name={} --package-version={} --json".format(name, version),
print_output=False,
)
assert code == 0 and json.loads(stdout).get("status") == "Added"
for file_path, name, version in dcos_files_list:
rc, out, err = sdk_cmd.run_cli("registry add --dcos-file={} --json".format(file_path))
assert rc == 0
assert len(json.loads(out)["packages"]) > 0, "No packages were added"
wait_for_added_registry(name, version)
def add_stub_universe_urls(stub_universe_urls: list) -> dict:
stub_urls = {}
if not stub_universe_urls:
return stub_urls
log.info('Adding stub URLs: {}'.format(stub_universe_urls))
for idx, url in enumerate(stub_universe_urls):
log.info('URL {}: {}'.format(idx, repr(url)))
package_name = 'testpkg-{}'.format(sdk_utils.random_string())
stub_urls[package_name] = url
# clean up any duplicate repositories
current_universes = sdk_cmd.run_cli('package repo list --json')
for repo in json.loads(current_universes)['repositories']:
if repo['uri'] in stub_urls.values():
log.info('Removing duplicate stub URL: {}'.format(repo['uri']))
sdk_cmd.run_cli('package repo remove {}'.format(repo['name']))
# add the needed universe repositories
for name, url in stub_urls.items():
log.info('Adding stub repo {} URL: {}'.format(name, url))
rc, stdout, stderr = sdk_cmd.run_raw_cli(
'package repo add --index=0 {} {}'.format(name, url))
if rc != 0:
raise Exception(
'Failed to add stub repo {} ({}): stdout=[{}], stderr=[{}]'.
format(name, url, stdout, stderr))
log.info('Finished adding universe repos')
return stub_urls
def add_dcos_files_to_registry(tmpdir_factory: TempdirFactory) -> None:
# Use DCOS_FILES_PATH if its set to a valid path OR use pytest's tmpdir.
dcos_files_path = os.environ.get("DCOS_FILES_PATH", "")
valid_path_set = os.path.isdir(dcos_files_path)
if valid_path_set and not os.access(dcos_files_path, os.W_OK):
log.warning("{} is not writable.".format(dcos_files_path))
valid_path_set = False
if not valid_path_set:
dcos_files_path = str(tmpdir_factory.mktemp(sdk_utils.random_string()))
stub_universe_urls = sdk_repository.get_repos()
log.info(
"Using {} to build .dcos files (if not exists) from {}".format(
dcos_files_path, stub_universe_urls
)
)
dcos_files_list = build_dcos_files_from_stubs(
stub_universe_urls, dcos_files_path, tmpdir_factory
)
log.info("Bundled .dcos files : {}".format(dcos_files_list))
@retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000)
def wait_for_added_registry(name: str, version: str) -> None:
code, stdout, stderr = sdk_cmd.run_cli(
"registry describe --package-name={} --package-version={} --json".format(name, version),
print_output=False,
)
assert code == 0 and json.loads(stdout).get("status") == "Added"
for file_path, name, version in dcos_files_list:
rc, out, err = sdk_cmd.run_cli("registry add --dcos-file={} --json".format(file_path))
assert rc == 0
assert len(json.loads(out)["packages"]) > 0, "No packages were added"
wait_for_added_registry(name, version)
def add_dcos_files_to_registry(
tmpdir_factory # _pytest.TempdirFactory
) -> None:
# Use DCOS_FILES_PATH if its set to a valid path OR use pytest's tmpdir.
dcos_files_path = os.environ.get('DCOS_FILES_PATH', '')
valid_path_set = os.path.isdir(dcos_files_path)
if valid_path_set and not os.access(dcos_files_path, os.W_OK):
log.warning('{} is not writable.'.format(dcos_files_path))
valid_path_set = False
if not valid_path_set:
dcos_files_path = str(tmpdir_factory.mktemp(sdk_utils.random_string()))
stub_universe_urls = sdk_repository.get_universe_repos()
log.info('Using {} to build .dcos files (if not exists) from {}'.format(
dcos_files_path, stub_universe_urls))
dcos_files_list = build_dcos_files_from_stubs(stub_universe_urls,
dcos_files_path,
tmpdir_factory)
log.info('Bundled .dcos files : {}'.format(dcos_files_list))
for file_path, name, version in dcos_files_list:
rc, out, err = sdk_cmd.run_raw_cli(' '.join(
['registry', 'add', '--dcos-file={}'.format(file_path), '--json']))
assert rc == 0
assert len(json.loads(out)['packages']) > 0, 'No packages were added'
wait_until_cli_condition(
' '.join([
'registry', 'describe', '--package-name=' + name,
'--package-version=' + version, '--json'
]), lambda code, out, err: code == 0 and json.loads(out).get(
'status') == 'Added')
def package_registry_session(tmpdir_factory): # _pytest.TempdirFactory
pkg_reg_repo = {}
service_uid = "pkg-reg-uid-{}".format(sdk_utils.random_string())
secret_path = "{}-secret-{}".format(service_uid, sdk_utils.random_string())
try:
sdk_security.create_service_account(service_uid, secret_path)
grant_perms_for_registry_account(service_uid)
pkg_reg_repo = install_package_registry(secret_path)
add_dcos_files_to_registry(tmpdir_factory)
yield
finally:
log.info("Teardown of package_registry_session initiated")
sdk_repository.remove_universe_repos(pkg_reg_repo)
# TODO If/when adding S3 backend, remove `Added` packages.
sdk_install.uninstall(PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME)
# No need to revoke perms, just delete the secret; the following ignores any failures.
sdk_security.delete_service_account(service_uid, secret_path)
def package_registry_session(tmpdir_factory: TempdirFactory) -> Iterator[None]:
pkg_reg_repo: Dict[str, str] = {}
service_uid = "pkg-reg-uid-{}".format(sdk_utils.random_string())
secret_path = "{}-secret-{}".format(service_uid, sdk_utils.random_string())
try:
sdk_security.create_service_account(service_uid, secret_path)
grant_perms_for_registry_account(service_uid)
pkg_reg_repo = install_package_registry(secret_path)
add_dcos_files_to_registry(tmpdir_factory)
yield
finally:
log.info("Teardown of package_registry_session initiated")
sdk_repository.remove_universe_repos(pkg_reg_repo)
# TODO If/when adding S3 backend, remove `Added` packages.
sdk_install.uninstall(PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME)
# No need to revoke perms, just delete the secret; the following ignores any failures.
sdk_security.delete_service_account(service_uid, secret_path)
def package_registry_session(tmpdir_factory): # _pytest.TempdirFactory
pkg_reg_stub = {}
pkg_reg_repo = {}
try:
# TODO Remove stub. We should install from bootstrap registry.
pkg_reg_stub = add_package_registry_stub()
service_uid = 'pkg-reg-uid-{}'.format(sdk_utils.random_string())
secret_path = '{}-secret-{}'.format(service_uid,
sdk_utils.random_string())
sdk_security.create_service_account(service_uid, secret_path)
grant_perms_for_registry_account(service_uid)
pkg_reg_repo = install_package_registry(secret_path)
add_dcos_files_to_registry(tmpdir_factory)
yield
finally:
log.info('Teardown of package_registry_session initiated')
sdk_repository.remove_universe_repos(pkg_reg_repo)
# TODO If/when adding S3 backend, remove `Added` packages.
sdk_install.uninstall(PACKAGE_REGISTRY_NAME,
PACKAGE_REGISTRY_SERVICE_NAME)
sdk_repository.remove_universe_repos(pkg_reg_stub)
# No need to revoke perms, just delete the secret.
sdk_security.delete_service_account(service_uid, secret_path)
def test_scaling_load(master_count, job_count, single_use: bool, run_delay,
cpu_quota, work_duration, mom, external_volume: bool,
scenario) -> None:
"""Launch a load test scenario. This does not verify the results
of the test, but does ensure the instances and jobs were created.
The installation is run in threads, but the job creation and
launch is then done serially after all Jenkins instances have
completed installation.
Args:
master_count: Number of Jenkins masters or instances
job_count: Number of Jobs on each Jenkins master
single_use: Mesos Single-Use Agent on (true) or off (false)
run_delay: Jobs should run every X minute(s)
cpu_quota: CPU quota (0.0 to disable)
work_duration: Time, in seconds, for generated jobs to sleep
mom: Marathon on Marathon instance name
external_volume: External volume on rexray (true) or local volume (false)
"""
with shakedown.marathon_on_marathon(mom):
if cpu_quota is not 0.0:
_setup_quota(SHARED_ROLE, cpu_quota)
masters = [
"jenkins{}".format(sdk_utils.random_string())
for _ in range(0, int(master_count))
]
# launch Jenkins services
install_threads = _spawn_threads(masters,
_install_jenkins,
external_volume=external_volume,
mom=mom,
daemon=True)
thread_failures = _wait_and_get_failures(install_threads,
timeout=DEPLOY_TIMEOUT)
thread_names = [x.name for x in thread_failures]
# the rest of the commands require a running Jenkins instance
deployed_masters = [x for x in masters if x not in thread_names]
job_threads = _spawn_threads(deployed_masters,
_create_jobs,
jobs=job_count,
single=single_use,
delay=run_delay,
duration=work_duration,
scenario=scenario)
_wait_on_threads(job_threads, JOB_RUN_TIMEOUT)
def create_slave() -> str:
label = sdk_utils.random_string()
jenkins.create_mesos_slave_node(label)
r = jenkins.create_mesos_slave_node(label,
service_name=config.SERVICE_NAME)
assert r.status_code == 200, 'create_mesos_slave_node failed : {}'.format(
r.status_code)
assert label in r.text, 'Label {} missing from {}'.format(label, r.text)
log.info("Set of labels is now: %s", r.text)
yield label
log.info("Removing label %s", label)
r = jenkins_remote_access.remove_slave_info(
label, service_name=config.SERVICE_NAME)
assert r.status_code == 200, 'remove_slave_info failed : {}'.format(
r.status_code)
assert label not in r.text, 'Label {} still present in {}'.format(
label, r.text)
log.info("Set of labels is now: %s", r.text)
def add_stub_universe_urls(stub_universe_urls: list) -> dict:
stub_urls = {}
if not stub_universe_urls:
return stub_urls
# clean up any duplicate repositories
_, current_universes, _ = sdk_cmd.run_cli("package repo list --json")
for repo in json.loads(current_universes)["repositories"]:
if repo["uri"] in stub_universe_urls:
log.info("Removing duplicate stub URL: {}".format(repo["uri"]))
assert remove_repo(repo["name"])
# add the needed universe repositories
log.info("Adding stub URLs: {}".format(stub_universe_urls))
for url in stub_universe_urls:
assert add_repo("testpkg-{}".format(sdk_utils.random_string()), url, 0)
return stub_urls
def add_stub_universe_urls(stub_universe_urls: List[str]) -> Dict[str, str]:
stub_urls: Dict[str, str] = {}
if not stub_universe_urls:
return stub_urls
# clean up any duplicate repositories
_, current_universes, _ = sdk_cmd.run_cli("package repo list --json")
for repo in json.loads(current_universes)["repositories"]:
if repo["uri"] in stub_universe_urls:
log.info("Removing duplicate stub URL: {}".format(repo["uri"]))
assert remove_repo(repo["name"])
# add the needed universe repositories
log.info("Adding stub URLs: {}".format(stub_universe_urls))
for url in stub_universe_urls:
name = "testpkg-{}".format(sdk_utils.random_string())
stub_urls[name] = url
assert add_repo(name, url, 0)
return stub_urls
def test_scaling_load(master_count, job_count, single_use: bool, run_delay,
cpu_quota, work_duration, mom, external_volume: bool,
scenario, min_index, max_index, batch_size) -> None:
"""Launch a load test scenario. This does not verify the results
of the test, but does ensure the instances and jobs were created.
The installation is run in threads, but the job creation and
launch is then done serially after all Jenkins instances have
completed installation.
Args:
master_count: Number of Jenkins masters or instances
job_count: Number of Jobs on each Jenkins master
single_use: Mesos Single-Use Agent on (true) or off (false)
run_delay: Jobs should run every X minute(s)
cpu_quota: CPU quota (0.0 to disable)
work_duration: Time, in seconds, for generated jobs to sleep
mom: Marathon on Marathon instance name
external_volume: External volume on rexray (true) or local volume (false)
min_index: minimum index to begin jenkins suffixes at
max_index: maximum index to end jenkins suffixes at
batch_size: batch size to deploy jenkins instances in
"""
security_mode = sdk_dcos.get_security_mode()
if mom and cpu_quota != 0.0:
with shakedown.marathon_on_marathon(mom):
_setup_quota(SHARED_ROLE, cpu_quota)
# create marathon client
if mom:
with shakedown.marathon_on_marathon(mom):
marathon_client = shakedown.marathon.create_client()
else:
marathon_client = shakedown.marathon.create_client()
masters = []
if min_index == -1 or max_index == -1:
masters = [
"jenkins{}".format(sdk_utils.random_string())
for _ in range(0, int(master_count))
]
else:
#max and min indexes are specified
#NOTE: using min/max will override master count
masters = [
"jenkins{}".format(index) for index in range(min_index, max_index)
]
# create service accounts in parallel
sdk_security.install_enterprise_cli()
service_account_threads = _spawn_threads(masters,
_create_service_accounts,
security=security_mode)
thread_failures = _wait_and_get_failures(service_account_threads,
timeout=SERVICE_ACCOUNT_TIMEOUT)
# launch Jenkins services
current = 0
end = max_index - min_index
while current + batch_size <= end:
log.info(
"Re-authenticating current batch load of jenkins{} - jenkins{} "
"to prevent auth-timeouts on scale cluster.".format(
current, current + batch_size))
dcos_login.login_session()
batched_masters = masters[current:current + batch_size]
install_threads = _spawn_threads(batched_masters,
_install_jenkins,
event='deployments',
client=marathon_client,
external_volume=external_volume,
security=security_mode,
daemon=True,
mom=mom)
thread_failures = _wait_and_get_failures(install_threads,
timeout=DEPLOY_TIMEOUT)
thread_names = [x.name for x in thread_failures]
# the rest of the commands require a running Jenkins instance
deployed_masters = [
x for x in batched_masters if x not in thread_names
]
job_threads = _spawn_threads(deployed_masters,
_create_jobs,
jobs=job_count,
single=single_use,
delay=run_delay,
duration=work_duration,
scenario=scenario)
_wait_on_threads(job_threads, JOB_RUN_TIMEOUT)
r = json.dumps(TIMINGS)
print(r)
current = current + batch_size
