def install(args):
options_file = args['--options-json']
if options_file:
if not os.path.isfile(options_file):
# TODO: Replace with logging
log.error("The specified file does not exist: %s", options_file)
sys.exit(1)
options = json.load(open(options_file, 'r'))
else:
options = get_default_options(args)
if args['--package-repo']:
sdk_repository.add_stub_universe_urls([args['--package-repo']])
rc, _, _ = sdk_cmd.run_raw_cli("package install spark --cli --yes")
assert rc == 0, "Error installing spark CLI"
quota_options = get_quota_options(args)
services = {}
services["spark"] = deploy_dispatchers(
num_dispatchers=int(args['<num_dispatchers>']),
service_name_base=args['<service_name_base>'],
output_file=args['<output_file>'],
linux_user=args["--user"],
options=options,
quota_options=quota_options)
output_filename = "{}-dispatchers.json".format(args["<output_file>"])
with open(output_filename, "w") as fp:
log.info("Saving dispatcher info to: %s", output_filename)
json.dump(services, fp, indent=2)
def install_package_registry(service_secret_path: str) -> Dict:
# Install Package Registry
# wait_for_deployment is `False` because the deployment checks do not apply
# to package registry as it is not an SDK app.
sdk_install.install(PACKAGE_REGISTRY_NAME,
PACKAGE_REGISTRY_SERVICE_NAME,
expected_running_tasks=0,
additional_options={
'registry': {
'service-account-secret-path':
service_secret_path
}
},
wait_for_deployment=False,
insert_strict_options=False)
log.info(
'{} with name {} installed successfully; ensuring its writable'.format(
PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME))
pkg_reg_repo = sdk_repository.add_stub_universe_urls([
'https://{}.marathon.l4lb.thisdcos.directory/repo'.format(
PACKAGE_REGISTRY_SERVICE_NAME)
])
# If `describe` endpoint is working, registry is writable by AR.
@retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000)
def wait_for_registry_available():
code, stdout, stderr = sdk_cmd.run_raw_cli(
'registry describe --package-name=hello --package-version=world')
assert code == 1 and 'Version [world] of package [hello] not found' in stderr
wait_for_registry_available()
return pkg_reg_repo
def install_package_registry(service_secret_path: str) -> Dict:
# Install Package Registry
# wait_for_deployment is `False` because the deployment checks do not apply
# to package registry as it is not an SDK app.
sdk_install.install(PACKAGE_REGISTRY_NAME,
PACKAGE_REGISTRY_SERVICE_NAME,
expected_running_tasks=0,
additional_options={
'registry': {
'service-account-secret-path':
service_secret_path
}
},
wait_for_deployment=False,
insert_strict_options=False)
log.info(
'{} with name {} installed successfully; ensuring its writable'.format(
PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME))
pkg_reg_repo = sdk_repository.add_stub_universe_urls([
'https://{}.marathon.l4lb.thisdcos.directory/repo'.format(
PACKAGE_REGISTRY_SERVICE_NAME)
])
# If `describe` endpoint is working, registry is writable by AR.
package_name = 'hello'
package_version = 'world'
expected_msg = 'Version [{}] of package [{}] ' \
'not found'.format(package_version, package_name)
wait_until_cli_condition(
'registry describe --package-name={} --package-version={}'.format(
package_name, package_version),
lambda code, out, err: code == 1 and expected_msg in err)
return pkg_reg_repo
def install_zookeeper_stub():
try:
zk_url = "https://universe-converter.mesosphere.com/transform?url=https://infinity-artifacts.s3.amazonaws.com/permanent/kafka-zookeeper/assets/sha-a0d96b28769e4cb871b3e2424f4c6b889f5a06dd/stub-universe-kafka-zookeeper.json"
stub_urls = sdk_repository.add_stub_universe_urls([zk_url, ])
yield
finally:
sdk_repository.remove_universe_repos(stub_urls)
def add_package_registry_stub() -> Dict:
# TODO Remove this method, install from bootstrap registry.
if PACKAGE_REGISTRY_STUB_URL not in os.environ:
raise Exception(
'{} is not found in env.'.format(PACKAGE_REGISTRY_STUB_URL))
stub_url = os.environ[PACKAGE_REGISTRY_STUB_URL]
with urllib.request.urlopen(stub_url) as url:
repo = json.loads(url.read().decode())
min_supported = [
x for x in repo['packages'] if x['name'] == PACKAGE_REGISTRY_NAME
][0]['minDcosReleaseVersion']
if sdk_utils.dcos_version_less_than(min_supported):
raise Exception(
'Min DC/OS {} required for package registry'.format(min_supported))
return sdk_repository.add_stub_universe_urls([stub_url])
def install_package_registry(service_secret_path: str) -> Dict[str, str]:
# If Bootstrap registry is not added by default and thus Readwrite registry is not found,
# fail the test.
code, _, _ = sdk_cmd.run_cli(
"package describe {}".format(PACKAGE_REGISTRY_NAME))
assert code == 0, "Package registry was not found to install. Exiting..."
# Install Package Registry
# wait_for_deployment is `False` because the deployment checks do not apply
# to package registry as it is not an SDK app.
sdk_install.install(
PACKAGE_REGISTRY_NAME,
PACKAGE_REGISTRY_SERVICE_NAME,
expected_running_tasks=0,
package_version=sdk_install.PackageVersion.LATEST_UNIVERSE,
additional_options={
"registry": {
"service-account-secret-path": service_secret_path
}
},
wait_for_deployment=False,
insert_strict_options=False,
)
log.info(
"{} with name {} installed successfully; ensuring its writable".format(
PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME))
pkg_reg_repo = sdk_repository.add_stub_universe_urls([
"https://{}.marathon.l4lb.thisdcos.directory/repo".format(
PACKAGE_REGISTRY_SERVICE_NAME)
])
# If `describe` endpoint is working, registry is writable by AR.
@retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000)
def wait_for_registry_available() -> None:
code, stdout, stderr = sdk_cmd.run_cli(
"registry describe --package-name=hello --package-version=world")
assert code == 1 and "Version [world] of package [hello] not found" in stderr
wait_for_registry_available()
return pkg_reg_repo
def install_package_registry(service_secret_path: str) -> Dict[str, str]:
# If Bootstrap registry is not added by default and thus Readwrite registry is not found,
# fail the test.
code, _, _ = sdk_cmd.run_cli("package describe {}".format(PACKAGE_REGISTRY_NAME))
assert code == 0, "Package registry was not found to install. Exiting..."
# Install Package Registry
# wait_for_deployment is `False` because the deployment checks do not apply
# to package registry as it is not an SDK app.
sdk_install.install(
PACKAGE_REGISTRY_NAME,
PACKAGE_REGISTRY_SERVICE_NAME,
expected_running_tasks=0,
package_version=sdk_install.PackageVersion.LATEST_UNIVERSE,
additional_options={"registry": {"service-account-secret-path": service_secret_path}},
wait_for_deployment=False,
insert_strict_options=False,
)
log.info(
"{} with name {} installed successfully; ensuring its writable".format(
PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME
)
)
pkg_reg_repo = sdk_repository.add_stub_universe_urls(
["https://{}.marathon.l4lb.thisdcos.directory/repo".format(PACKAGE_REGISTRY_SERVICE_NAME)]
)
# If `describe` endpoint is working, registry is writable by AR.
@retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000)
def wait_for_registry_available() -> None:
code, stdout, stderr = sdk_cmd.run_cli(
"registry describe --package-name=hello --package-version=world"
)
assert code == 1 and "Version [world] of package [hello] not found" in stderr
wait_for_registry_available()
return pkg_reg_repo
