def wait_for_completed_recovery(service_name, timeout_seconds=None): def fn(): return recovery_plan_is_finished(service_name) if timeout_seconds: return sdk_spin.time_wait_return(fn, timeout_seconds=timeout_seconds) else: return sdk_spin.time_wait_return(fn)
def broker_count_check(count): def fun(): try: if len(service_cli('broker list')) == count: return True except: pass return False spin.time_wait_return(fun)
def get_plan(service_name, plan): def fn(): response = dcos.http.get("{}/v1/plans/{}".format(shakedown.dcos_service_url(service_name), plan)) response.raise_for_status() return response return sdk_spin.time_wait_return(lambda: fn())
def get_dcos_cassandra_plan(service_name): utils.out('Waiting for {} plan to complete...'.format(service_name)) def fn(): return api.get(service_name, '/v1/plan') return spin.time_wait_return(fn)
def test_marathon_rack_not_found(): def fun(): try: return service_cli('plan show {}'.format(DEFAULT_PLAN_NAME)) except: return False shakedown.install_package( PACKAGE_NAME, service_name=SERVICE_NAME, options_json=install.get_package_options(additional_options={ 'service': { 'placement_constraint': 'rack_id:LIKE:rack-foo-.*' } }), wait_for_completion=False) try: tasks.check_running(PACKAGE_NAME, 1, timeout_seconds=120) assert False, "Should have failed to install" except AssertionError as arg: raise arg except: pass # expected to fail pl = spin.time_wait_return(fun) # check that first node is still (unsuccessfully) looking for a match: assert pl['status'] == 'IN_PROGRESS' assert pl['phases'][0]['status'] == 'IN_PROGRESS' # if so early, it can be PREPARED ? assert pl['phases'][0]['steps'][0]['status'] in ('PREPARED', 'PENDING') assert pl['phases'][0]['steps'][1]['status'] == 'PENDING' assert pl['phases'][0]['steps'][2]['status'] == 'PENDING' install.uninstall(SERVICE_NAME, PACKAGE_NAME)
def test_marathon_rack_not_found(): def fun(): try: return service_cli('plan show {}'.format(DEFAULT_PLAN_NAME)) except: return False shakedown.install_package(PACKAGE_NAME, service_name=SERVICE_NAME, options_json=install.get_package_options( additional_options={'service':{'placement_constraint':'rack_id:LIKE:rack-foo-.*'}} ), wait_for_completion=False) try: tasks.check_running(PACKAGE_NAME, 1, timeout_seconds=120) assert False, "Should have failed to install" except AssertionError as arg: raise arg except: pass # expected to fail pl = spin.time_wait_return(fun) # check that first node is still (unsuccessfully) looking for a match: assert pl['status'] == 'IN_PROGRESS' assert pl['phases'][0]['status'] == 'IN_PROGRESS' # if so early, it can be PREPARED ? assert pl['phases'][0]['steps'][0]['status'] in ('PREPARED', 'PENDING') assert pl['phases'][0]['steps'][1]['status'] == 'PENDING' assert pl['phases'][0]['steps'][2]['status'] == 'PENDING' install.uninstall(SERVICE_NAME, PACKAGE_NAME)
def get_plan(service_name, plan): sdk_utils.out("Waiting for {} plan to complete...".format(service_name)) def fn(): return sdk_api.get(service_name, "/v1/plans/{}".format(plan)) return sdk_spin.time_wait_return(fn)
def service_plan_wait(): def fun(): try: return cmd.run_cli('hello-world plan show deploy') except: return False return spin.time_wait_return(fun)
def service_plan_wait(plan_name): def fun(): try: return service_cli('plan show {}'.format(plan_name)) except: return False return spin.time_wait_return(fun)
def get_config(app_name): def fn(): return sdk_cmd.request('get', api_url('apps/{}'.format(app_name)), retry=False) config = sdk_spin.time_wait_return(lambda: fn()).json()['app'] del config['uris'] del config['version'] return config
def test_endpoints_address(): def fun(): ret = service_cli('endpoints {}'.format(DEFAULT_TASK_NAME)) if len(ret['address']) == DEFAULT_BROKER_COUNT: return ret return False address = spin.time_wait_return(fun) assert len(address) == 3 assert len(address['dns']) == DEFAULT_BROKER_COUNT
def test_endpoints_address(): def fun(): ret = service_cli('endpoints {}'.format(DEFAULT_TASK_NAME)) if len(ret['address']) == DEFAULT_BROKER_COUNT: return ret return False endpoints = spin.time_wait_return(fun) # NOTE: do NOT closed-to-extension assert len(endpoints) == _something_ assert len(endpoints['address']) == DEFAULT_BROKER_COUNT assert len(endpoints['dns']) == DEFAULT_BROKER_COUNT
def request(method, url, retry=True, **kwargs): def fn(): response = dcos.http.request(method, url, **kwargs) print('Got {} for {} {} (args: {})'.format( response.status_code, method.upper(), url, kwargs)) response.raise_for_status() return response if retry: return sdk_spin.time_wait_return(lambda: fn()) else: return fn()
def request(method, url, retry=True, **kwargs): def fn(): response = dcos.http.request(method, url, **kwargs) sdk_utils.out('Got {} for {} {} (args: {})'.format( response.status_code, method.upper(), url, kwargs)) response.raise_for_status() return response if retry: return sdk_spin.time_wait_return(lambda: fn()) else: return fn()
def get_config(app_name): def fn(): return sdk_cmd.request('get', api_url('apps/{}'.format(app_name)), retry=False) config = sdk_spin.time_wait_return(lambda: fn()).json()['app'] # The configuration JSON that marathon returns doesn't match the configuration JSON it accepts, # so we have to remove some offending fields to make it re-submittable, since it's not possible to # submit a partial config with only the desired fields changed. del config['uris'] del config['version'] return config
def service_plan_complete(plan_name): def fun(): try: pl = service_cli('plan show {}'.format(plan_name)) print( 'Running service_plan_complete for plan {}'.format(plan_name)) print(pl) print('status = {}'.format(pl['status'])) if pl['status'] == 'COMPLETE': return True except: traceback.print_exc() return False print('Plan {} is not complete ({})'.format(plan_name, pl['status'])) return False return spin.time_wait_return(fun)
def test_state_refresh_disable_cache(): '''Disables caching via a scheduler envvar''' check_running() task_ids = tasks.get_task_ids(PACKAGE_NAME, '') # caching enabled by default: stdout = cmd.run_cli('hello-world state refresh_cache') assert "Received cmd: refresh" in stdout config = marathon.get_config(PACKAGE_NAME) cpus = float(config['env']['HELLO_CPUS']) config['env']['DISABLE_STATE_CACHE'] = 'any-text-here' cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config) tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids) check_running() # caching disabled, refresh_cache should fail with a 409 error (eventually, once scheduler is up): def check_cache_refresh_fails_409conflict(): try: cmd.run_cli('hello-world state refresh_cache') except Exception as e: if "failed: 409 Conflict" in e.args[0]: return True return False spin.time_wait_noisy(lambda: check_cache_refresh_fails_409conflict(), timeout_seconds=120.) config = marathon.get_config(PACKAGE_NAME) cpus = float(config['env']['HELLO_CPUS']) del config['env']['DISABLE_STATE_CACHE'] cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config) tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids) check_running() # caching reenabled, refresh_cache should succeed (eventually, once scheduler is up): def check_cache_refresh(): return cmd.run_cli('hello-world state refresh_cache') stdout = spin.time_wait_return(lambda: check_cache_refresh(), timeout_seconds=120.) assert "Received cmd: refresh" in stdout
def test_increase_cpu(): def plan_waiting(): try: pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME)) if pl['status'] == 'WAITING': return True except: pass return False def plan_complete(): try: pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME)) if pl['status'] == 'COMPLETE': return True except: pass return False config = marathon.get_config(SERVICE_NAME) config['env']['BROKER_CPUS'] = str(0.1 + float(config['env']['BROKER_CPUS'])) marathon.update_app(SERVICE_NAME, config) spin.time_wait_return(plan_waiting) pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME)) assert pl['status'] == 'WAITING' assert pl['phases'][0]['status'] == 'WAITING' assert pl['phases'][0]['steps'][0]['status'] == 'WAITING' assert pl['phases'][0]['steps'][1]['status'] == 'WAITING' for step in range (2, DEFAULT_BROKER_COUNT +1 ): assert pl['phases'][0]['steps'][step]['status'] == 'PENDING' # all tasks are still running tasks.check_running(SERVICE_NAME, DEFAULT_BROKER_COUNT + 1) broker_ids = tasks.get_task_ids(SERVICE_NAME, '{}-0-{}'.format(DEFAULT_POD_TYPE, DEFAULT_TASK_NAME)) service_cli('plan continue {} {}'.format(DEFAULT_PLAN_NAME, DEFAULT_PHASE_NAME)) tasks.check_tasks_updated(SERVICE_NAME, '{}-0-{}'.format(DEFAULT_POD_TYPE, DEFAULT_TASK_NAME), broker_ids) tasks.check_running(SERVICE_NAME, DEFAULT_BROKER_COUNT + 1) pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME)) assert pl['status'] == 'WAITING' assert pl['phases'][0]['status'] == 'WAITING' assert pl['phases'][0]['steps'][0]['status'] == 'COMPLETE' assert pl['phases'][0]['steps'][1]['status'] == 'WAITING' for step in range(2, DEFAULT_BROKER_COUNT + 1): assert pl['phases'][0]['steps'][step]['status'] == 'PENDING' broker_ids = tasks.get_task_ids(SERVICE_NAME, '{}-1-{}'.format(DEFAULT_POD_TYPE, DEFAULT_TASK_NAME)) service_cli('plan continue {} {}'.format(DEFAULT_PLAN_NAME, DEFAULT_PHASE_NAME)) tasks.check_tasks_updated(SERVICE_NAME, '{}-1-{}'.format(DEFAULT_POD_TYPE, DEFAULT_TASK_NAME), broker_ids) spin.time_wait_return(plan_complete) pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME)) assert pl['status'] == 'COMPLETE' assert pl['phases'][0]['status'] == 'COMPLETE' for step in range(DEFAULT_BROKER_COUNT + 1): assert pl['phases'][0]['steps'][step]['status'] == 'COMPLETE' broker_count_check(DEFAULT_BROKER_COUNT + 1)
def get_plan(service_name, plan): def fn(): return sdk_api.get(service_name, "/v1/plans/{}".format(plan)) return sdk_spin.time_wait_return(fn)
def wait_for_completed_deployment(service_name): def fn(): return deployment_plan_is_finished(service_name) return sdk_spin.time_wait_return(fn)
def wait_for_completed_recovery(service_name): def fn(): return recovery_plan_is_finished(service_name) return sdk_spin.time_wait_return(fn)