def install(args): options_file = args['--options-json'] if options_file: if not os.path.isfile(options_file): # TODO: Replace with logging log.error("The specified file does not exist: %s", options_file) sys.exit(1) options = json.load(open(options_file, 'r')) else: options = get_default_options(args) if args['--package-repo']: sdk_repository.add_stub_universe_urls([args['--package-repo']]) rc, _, _ = sdk_cmd.run_raw_cli("package install spark --cli --yes") assert rc == 0, "Error installing spark CLI" quota_options = get_quota_options(args) services = {} services["spark"] = deploy_dispatchers( num_dispatchers=int(args['<num_dispatchers>']), service_name_base=args['<service_name_base>'], output_file=args['<output_file>'], linux_user=args["--user"], options=options, quota_options=quota_options) output_filename = "{}-dispatchers.json".format(args["<output_file>"]) with open(output_filename, "w") as fp: log.info("Saving dispatcher info to: %s", output_filename) json.dump(services, fp, indent=2)
def install_package_registry(service_secret_path: str) -> Dict: # Install Package Registry # wait_for_deployment is `False` because the deployment checks do not apply # to package registry as it is not an SDK app. sdk_install.install(PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME, expected_running_tasks=0, additional_options={ 'registry': { 'service-account-secret-path': service_secret_path } }, wait_for_deployment=False, insert_strict_options=False) log.info( '{} with name {} installed successfully; ensuring its writable'.format( PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME)) pkg_reg_repo = sdk_repository.add_stub_universe_urls([ 'https://{}.marathon.l4lb.thisdcos.directory/repo'.format( PACKAGE_REGISTRY_SERVICE_NAME) ]) # If `describe` endpoint is working, registry is writable by AR. @retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000) def wait_for_registry_available(): code, stdout, stderr = sdk_cmd.run_raw_cli( 'registry describe --package-name=hello --package-version=world') assert code == 1 and 'Version [world] of package [hello] not found' in stderr wait_for_registry_available() return pkg_reg_repo
def install_package_registry(service_secret_path: str) -> Dict: # Install Package Registry # wait_for_deployment is `False` because the deployment checks do not apply # to package registry as it is not an SDK app. sdk_install.install(PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME, expected_running_tasks=0, additional_options={ 'registry': { 'service-account-secret-path': service_secret_path } }, wait_for_deployment=False, insert_strict_options=False) log.info( '{} with name {} installed successfully; ensuring its writable'.format( PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME)) pkg_reg_repo = sdk_repository.add_stub_universe_urls([ 'https://{}.marathon.l4lb.thisdcos.directory/repo'.format( PACKAGE_REGISTRY_SERVICE_NAME) ]) # If `describe` endpoint is working, registry is writable by AR. package_name = 'hello' package_version = 'world' expected_msg = 'Version [{}] of package [{}] ' \ 'not found'.format(package_version, package_name) wait_until_cli_condition( 'registry describe --package-name={} --package-version={}'.format( package_name, package_version), lambda code, out, err: code == 1 and expected_msg in err) return pkg_reg_repo
def install_zookeeper_stub(): try: zk_url = "https://universe-converter.mesosphere.com/transform?url=https://infinity-artifacts.s3.amazonaws.com/permanent/kafka-zookeeper/assets/sha-a0d96b28769e4cb871b3e2424f4c6b889f5a06dd/stub-universe-kafka-zookeeper.json" stub_urls = sdk_repository.add_stub_universe_urls([zk_url, ]) yield finally: sdk_repository.remove_universe_repos(stub_urls)
def add_package_registry_stub() -> Dict: # TODO Remove this method, install from bootstrap registry. if PACKAGE_REGISTRY_STUB_URL not in os.environ: raise Exception( '{} is not found in env.'.format(PACKAGE_REGISTRY_STUB_URL)) stub_url = os.environ[PACKAGE_REGISTRY_STUB_URL] with urllib.request.urlopen(stub_url) as url: repo = json.loads(url.read().decode()) min_supported = [ x for x in repo['packages'] if x['name'] == PACKAGE_REGISTRY_NAME ][0]['minDcosReleaseVersion'] if sdk_utils.dcos_version_less_than(min_supported): raise Exception( 'Min DC/OS {} required for package registry'.format(min_supported)) return sdk_repository.add_stub_universe_urls([stub_url])
def install_package_registry(service_secret_path: str) -> Dict[str, str]: # If Bootstrap registry is not added by default and thus Readwrite registry is not found, # fail the test. code, _, _ = sdk_cmd.run_cli( "package describe {}".format(PACKAGE_REGISTRY_NAME)) assert code == 0, "Package registry was not found to install. Exiting..." # Install Package Registry # wait_for_deployment is `False` because the deployment checks do not apply # to package registry as it is not an SDK app. sdk_install.install( PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME, expected_running_tasks=0, package_version=sdk_install.PackageVersion.LATEST_UNIVERSE, additional_options={ "registry": { "service-account-secret-path": service_secret_path } }, wait_for_deployment=False, insert_strict_options=False, ) log.info( "{} with name {} installed successfully; ensuring its writable".format( PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME)) pkg_reg_repo = sdk_repository.add_stub_universe_urls([ "https://{}.marathon.l4lb.thisdcos.directory/repo".format( PACKAGE_REGISTRY_SERVICE_NAME) ]) # If `describe` endpoint is working, registry is writable by AR. @retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000) def wait_for_registry_available() -> None: code, stdout, stderr = sdk_cmd.run_cli( "registry describe --package-name=hello --package-version=world") assert code == 1 and "Version [world] of package [hello] not found" in stderr wait_for_registry_available() return pkg_reg_repo
def install_package_registry(service_secret_path: str) -> Dict[str, str]: # If Bootstrap registry is not added by default and thus Readwrite registry is not found, # fail the test. code, _, _ = sdk_cmd.run_cli("package describe {}".format(PACKAGE_REGISTRY_NAME)) assert code == 0, "Package registry was not found to install. Exiting..." # Install Package Registry # wait_for_deployment is `False` because the deployment checks do not apply # to package registry as it is not an SDK app. sdk_install.install( PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME, expected_running_tasks=0, package_version=sdk_install.PackageVersion.LATEST_UNIVERSE, additional_options={"registry": {"service-account-secret-path": service_secret_path}}, wait_for_deployment=False, insert_strict_options=False, ) log.info( "{} with name {} installed successfully; ensuring its writable".format( PACKAGE_REGISTRY_NAME, PACKAGE_REGISTRY_SERVICE_NAME ) ) pkg_reg_repo = sdk_repository.add_stub_universe_urls( ["https://{}.marathon.l4lb.thisdcos.directory/repo".format(PACKAGE_REGISTRY_SERVICE_NAME)] ) # If `describe` endpoint is working, registry is writable by AR. @retrying.retry(stop_max_delay=5 * 60 * 1000, wait_fixed=5 * 1000) def wait_for_registry_available() -> None: code, stdout, stderr = sdk_cmd.run_cli( "registry describe --package-name=hello --package-version=world" ) assert code == 1 and "Version [world] of package [hello] not found" in stderr wait_for_registry_available() return pkg_reg_repo