def deploy_dispatchers(num_dispatchers, service_name_base, output_file, options, options_file=None, package_repo=None, quota_dispatcher_cpus=1, quota_dispatcher_gpus=0, quota_dispatcher_mem=2048.0, quota_driver_cpus=1, quota_driver_gpus=0, quota_driver_mem=1024.0): with open(output_file, "w") as outfile: shakedown.run_dcos_command("package install spark --cli --yes") for i in range(0, num_dispatchers): service_name = "{}-{}".format(service_name_base, str(i)) # set service name options["service"]["name"] = service_name if package_repo is not None: if package_repo not in [ x['uri'] for x in shakedown.get_package_repos()['repositories'] ]: shakedown.add_package_repo( repo_name="{}-repo".format(service_name_base), repo_url=package_repo) # create dispatcher & driver role quotas dispatcher_role = "{}-dispatcher-role".format(service_name) create_quota(name=dispatcher_role, cpus=quota_dispatcher_cpus, gpus=quota_dispatcher_gpus, mem=quota_dispatcher_mem) driver_role = "{}-driver-role".format(service_name) create_quota(name=driver_role, cpus=quota_driver_cpus, gpus=quota_driver_gpus, mem=quota_driver_mem) # install dispatcher with appropriate role options["service"]["role"] = dispatcher_role if options_file is not None: shakedown.install_package( package_name=arguments['--package-name'], service_name=service_name, options_file=options_file) else: shakedown.install_package( package_name=arguments['--package-name'], service_name=service_name, options_json=options) outfile.write("{},{},{}\n".format(service_name, dispatcher_role, driver_role))
def test_upgrade( package_name, service_name, running_task_count, additional_options={}, test_version_additional_options=None, timeout_seconds=25*60, wait_for_deployment=True): # allow providing different options dicts to the universe version vs the test version: if test_version_additional_options is None: test_version_additional_options = additional_options sdk_install.uninstall(package_name, service_name) test_version = _get_pkg_version(package_name) log.info('Found test version: {}'.format(test_version)) universe_url = _get_universe_url() universe_version = None try: # Move the Universe repo to the top of the repo list so that we can first install the release version. shakedown.remove_package_repo('Universe') assert shakedown.add_package_repo('Universe', universe_url, 0) log.info('Waiting for Universe release version of {} to appear: version != {}'.format( package_name, test_version)) universe_version = _wait_for_new_package_version(package_name, test_version) log.info('Installing Universe version: {}={}'.format(package_name, universe_version)) sdk_install.install( package_name, service_name, running_task_count, additional_options=additional_options, timeout_seconds=timeout_seconds, wait_for_deployment=wait_for_deployment) finally: if universe_version: # Return the Universe repo back to the bottom of the repo list so that we can upgrade to the build version. shakedown.remove_package_repo('Universe') assert shakedown.add_package_repo('Universe', universe_url) log.info('Waiting for test build version of {} to appear: version != {}'.format( package_name, universe_version)) _wait_for_new_package_version(package_name, universe_version) log.info('Upgrading {}: {} => {}'.format(package_name, universe_version, test_version)) _upgrade_or_downgrade( package_name, test_version, service_name, running_task_count, test_version_additional_options, timeout_seconds, wait_for_deployment)
def add_repo(repo_name, repo_url, prev_version): assert shakedown.add_package_repo( repo_name, repo_url, 0) # Make sure the new repo packages are available new_default_version_available(prev_version)
def _add_repo(repo_name, repo_url, prev_version, index, default_repo_package_name): assert shakedown.add_package_repo( repo_name, repo_url, index) # Make sure the new default repo packages are available _wait_for_new_default_version(prev_version, default_repo_package_name)
def add_repo(prev_version): assert shakedown.add_package_repo(MASTER_CUSTOM_NAME, MASTER_CUSTOM_URL, 0) # Make sure the new repo packages are available new_default_version_available(prev_version)
def add_last_repo(repo_name, repo_url, prev_version, default_repo_package_name): assert shakedown.add_package_repo(repo_name, repo_url) # Make sure the new default repo packages are available new_default_version_available(prev_version, default_repo_package_name)
def add_repo(repo_name, repo_url, prev_version): assert shakedown.add_package_repo(repo_name, repo_url, 0) # Make sure the new repo packages are available # The above invocation's effects don't occur immediately # so we spin until they do new_default_version_available(prev_version)