def deploy_dispatchers(num_dispatchers,
service_name_base,
output_file,
options,
options_file=None,
package_repo=None,
quota_dispatcher_cpus=1,
quota_dispatcher_gpus=0,
quota_dispatcher_mem=2048.0,
quota_driver_cpus=1,
quota_driver_gpus=0,
quota_driver_mem=1024.0):
with open(output_file, "w") as outfile:
shakedown.run_dcos_command("package install spark --cli --yes")
for i in range(0, num_dispatchers):
service_name = "{}-{}".format(service_name_base, str(i))
# set service name
options["service"]["name"] = service_name
if package_repo is not None:
if package_repo not in [
x['uri']
for x in shakedown.get_package_repos()['repositories']
]:
shakedown.add_package_repo(
repo_name="{}-repo".format(service_name_base),
repo_url=package_repo)
# create dispatcher & driver role quotas
dispatcher_role = "{}-dispatcher-role".format(service_name)
create_quota(name=dispatcher_role,
cpus=quota_dispatcher_cpus,
gpus=quota_dispatcher_gpus,
mem=quota_dispatcher_mem)
driver_role = "{}-driver-role".format(service_name)
create_quota(name=driver_role,
cpus=quota_driver_cpus,
gpus=quota_driver_gpus,
mem=quota_driver_mem)
# install dispatcher with appropriate role
options["service"]["role"] = dispatcher_role
if options_file is not None:
shakedown.install_package(
package_name=arguments['--package-name'],
service_name=service_name,
options_file=options_file)
else:
shakedown.install_package(
package_name=arguments['--package-name'],
service_name=service_name,
options_json=options)
outfile.write("{},{},{}\n".format(service_name, dispatcher_role,
driver_role))
def test_upgrade(
package_name,
service_name,
running_task_count,
additional_options={},
test_version_additional_options=None,
timeout_seconds=25*60,
wait_for_deployment=True):
# allow providing different options dicts to the universe version vs the test version:
if test_version_additional_options is None:
test_version_additional_options = additional_options
sdk_install.uninstall(package_name, service_name)
test_version = _get_pkg_version(package_name)
log.info('Found test version: {}'.format(test_version))
universe_url = _get_universe_url()
universe_version = None
try:
# Move the Universe repo to the top of the repo list so that we can first install the release version.
shakedown.remove_package_repo('Universe')
assert shakedown.add_package_repo('Universe', universe_url, 0)
log.info('Waiting for Universe release version of {} to appear: version != {}'.format(
package_name, test_version))
universe_version = _wait_for_new_package_version(package_name, test_version)
log.info('Installing Universe version: {}={}'.format(package_name, universe_version))
sdk_install.install(
package_name,
service_name,
running_task_count,
additional_options=additional_options,
timeout_seconds=timeout_seconds,
wait_for_deployment=wait_for_deployment)
finally:
if universe_version:
# Return the Universe repo back to the bottom of the repo list so that we can upgrade to the build version.
shakedown.remove_package_repo('Universe')
assert shakedown.add_package_repo('Universe', universe_url)
log.info('Waiting for test build version of {} to appear: version != {}'.format(
package_name, universe_version))
_wait_for_new_package_version(package_name, universe_version)
log.info('Upgrading {}: {} => {}'.format(package_name, universe_version, test_version))
_upgrade_or_downgrade(
package_name,
test_version,
service_name,
running_task_count,
test_version_additional_options,
timeout_seconds,
wait_for_deployment)
def add_repo(repo_name, repo_url, prev_version):
assert shakedown.add_package_repo(
repo_name,
repo_url,
0)
# Make sure the new repo packages are available
new_default_version_available(prev_version)
def add_repo(repo_name, repo_url, prev_version):
assert shakedown.add_package_repo(
repo_name,
repo_url,
0)
# Make sure the new repo packages are available
new_default_version_available(prev_version)
def _add_repo(repo_name, repo_url, prev_version, index, default_repo_package_name):
assert shakedown.add_package_repo(
repo_name,
repo_url,
index)
# Make sure the new default repo packages are available
_wait_for_new_default_version(prev_version, default_repo_package_name)
def _add_repo(repo_name, repo_url, prev_version, index, default_repo_package_name):
assert shakedown.add_package_repo(
repo_name,
repo_url,
index)
# Make sure the new default repo packages are available
_wait_for_new_default_version(prev_version, default_repo_package_name)
def add_repo(prev_version):
assert shakedown.add_package_repo(MASTER_CUSTOM_NAME, MASTER_CUSTOM_URL, 0)
# Make sure the new repo packages are available
new_default_version_available(prev_version)
def add_last_repo(repo_name, repo_url, prev_version,
default_repo_package_name):
assert shakedown.add_package_repo(repo_name, repo_url)
# Make sure the new default repo packages are available
new_default_version_available(prev_version, default_repo_package_name)
def add_repo(repo_name, repo_url, prev_version):
assert shakedown.add_package_repo(repo_name, repo_url, 0)
# Make sure the new repo packages are available
# The above invocation's effects don't occur immediately
# so we spin until they do
new_default_version_available(prev_version)
