def wait_for_completed_recovery(service_name, timeout_seconds=None):
def fn():
return recovery_plan_is_finished(service_name)
if timeout_seconds:
return sdk_spin.time_wait_return(fn, timeout_seconds=timeout_seconds)
else:
return sdk_spin.time_wait_return(fn)
def broker_count_check(count):
def fun():
try:
if len(service_cli('broker list')) == count:
return True
except:
pass
return False
spin.time_wait_return(fun)
def get_plan(service_name, plan):
def fn():
response = dcos.http.get("{}/v1/plans/{}".format(shakedown.dcos_service_url(service_name), plan))
response.raise_for_status()
return response
return sdk_spin.time_wait_return(lambda: fn())
def get_dcos_cassandra_plan(service_name):
utils.out('Waiting for {} plan to complete...'.format(service_name))
def fn():
return api.get(service_name, '/v1/plan')
return spin.time_wait_return(fn)
def test_marathon_rack_not_found():
def fun():
try:
return service_cli('plan show {}'.format(DEFAULT_PLAN_NAME))
except:
return False
shakedown.install_package(
PACKAGE_NAME,
service_name=SERVICE_NAME,
options_json=install.get_package_options(additional_options={
'service': {
'placement_constraint': 'rack_id:LIKE:rack-foo-.*'
}
}),
wait_for_completion=False)
try:
tasks.check_running(PACKAGE_NAME, 1, timeout_seconds=120)
assert False, "Should have failed to install"
except AssertionError as arg:
raise arg
except:
pass # expected to fail
pl = spin.time_wait_return(fun)
# check that first node is still (unsuccessfully) looking for a match:
assert pl['status'] == 'IN_PROGRESS'
assert pl['phases'][0]['status'] == 'IN_PROGRESS'
# if so early, it can be PREPARED ?
assert pl['phases'][0]['steps'][0]['status'] in ('PREPARED', 'PENDING')
assert pl['phases'][0]['steps'][1]['status'] == 'PENDING'
assert pl['phases'][0]['steps'][2]['status'] == 'PENDING'
install.uninstall(SERVICE_NAME, PACKAGE_NAME)
def test_marathon_rack_not_found():
def fun():
try:
return service_cli('plan show {}'.format(DEFAULT_PLAN_NAME))
except:
return False
shakedown.install_package(PACKAGE_NAME,
service_name=SERVICE_NAME,
options_json=install.get_package_options(
additional_options={'service':{'placement_constraint':'rack_id:LIKE:rack-foo-.*'}}
),
wait_for_completion=False)
try:
tasks.check_running(PACKAGE_NAME, 1, timeout_seconds=120)
assert False, "Should have failed to install"
except AssertionError as arg:
raise arg
except:
pass # expected to fail
pl = spin.time_wait_return(fun)
# check that first node is still (unsuccessfully) looking for a match:
assert pl['status'] == 'IN_PROGRESS'
assert pl['phases'][0]['status'] == 'IN_PROGRESS'
# if so early, it can be PREPARED ?
assert pl['phases'][0]['steps'][0]['status'] in ('PREPARED', 'PENDING')
assert pl['phases'][0]['steps'][1]['status'] == 'PENDING'
assert pl['phases'][0]['steps'][2]['status'] == 'PENDING'
install.uninstall(SERVICE_NAME, PACKAGE_NAME)
def get_plan(service_name, plan):
sdk_utils.out("Waiting for {} plan to complete...".format(service_name))
def fn():
return sdk_api.get(service_name, "/v1/plans/{}".format(plan))
return sdk_spin.time_wait_return(fn)
def service_plan_wait():
def fun():
try:
return cmd.run_cli('hello-world plan show deploy')
except:
return False
return spin.time_wait_return(fun)
def service_plan_wait(plan_name):
def fun():
try:
return service_cli('plan show {}'.format(plan_name))
except:
return False
return spin.time_wait_return(fun)
def get_config(app_name):
def fn():
return sdk_cmd.request('get', api_url('apps/{}'.format(app_name)), retry=False)
config = sdk_spin.time_wait_return(lambda: fn()).json()['app']
del config['uris']
del config['version']
return config
def get_config(app_name):
def fn():
return sdk_cmd.request('get', api_url('apps/{}'.format(app_name)), retry=False)
config = sdk_spin.time_wait_return(lambda: fn()).json()['app']
del config['uris']
del config['version']
return config
def test_endpoints_address():
def fun():
ret = service_cli('endpoints {}'.format(DEFAULT_TASK_NAME))
if len(ret['address']) == DEFAULT_BROKER_COUNT:
return ret
return False
address = spin.time_wait_return(fun)
assert len(address) == 3
assert len(address['dns']) == DEFAULT_BROKER_COUNT
def test_endpoints_address():
def fun():
ret = service_cli('endpoints {}'.format(DEFAULT_TASK_NAME))
if len(ret['address']) == DEFAULT_BROKER_COUNT:
return ret
return False
endpoints = spin.time_wait_return(fun)
# NOTE: do NOT closed-to-extension assert len(endpoints) == _something_
assert len(endpoints['address']) == DEFAULT_BROKER_COUNT
assert len(endpoints['dns']) == DEFAULT_BROKER_COUNT
def request(method, url, retry=True, **kwargs):
def fn():
response = dcos.http.request(method, url, **kwargs)
print('Got {} for {} {} (args: {})'.format(
response.status_code, method.upper(), url, kwargs))
response.raise_for_status()
return response
if retry:
return sdk_spin.time_wait_return(lambda: fn())
else:
return fn()
def request(method, url, retry=True, **kwargs):
def fn():
response = dcos.http.request(method, url, **kwargs)
sdk_utils.out('Got {} for {} {} (args: {})'.format(
response.status_code, method.upper(), url, kwargs))
response.raise_for_status()
return response
if retry:
return sdk_spin.time_wait_return(lambda: fn())
else:
return fn()
def get_config(app_name):
def fn():
return sdk_cmd.request('get', api_url('apps/{}'.format(app_name)), retry=False)
config = sdk_spin.time_wait_return(lambda: fn()).json()['app']
# The configuration JSON that marathon returns doesn't match the configuration JSON it accepts,
# so we have to remove some offending fields to make it re-submittable, since it's not possible to
# submit a partial config with only the desired fields changed.
del config['uris']
del config['version']
return config
def service_plan_complete(plan_name):
def fun():
try:
pl = service_cli('plan show {}'.format(plan_name))
print(
'Running service_plan_complete for plan {}'.format(plan_name))
print(pl)
print('status = {}'.format(pl['status']))
if pl['status'] == 'COMPLETE':
return True
except:
traceback.print_exc()
return False
print('Plan {} is not complete ({})'.format(plan_name, pl['status']))
return False
return spin.time_wait_return(fun)
def test_state_refresh_disable_cache():
'''Disables caching via a scheduler envvar'''
check_running()
task_ids = tasks.get_task_ids(PACKAGE_NAME, '')
# caching enabled by default:
stdout = cmd.run_cli('hello-world state refresh_cache')
assert "Received cmd: refresh" in stdout
config = marathon.get_config(PACKAGE_NAME)
cpus = float(config['env']['HELLO_CPUS'])
config['env']['DISABLE_STATE_CACHE'] = 'any-text-here'
cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config)
tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids)
check_running()
# caching disabled, refresh_cache should fail with a 409 error (eventually, once scheduler is up):
def check_cache_refresh_fails_409conflict():
try:
cmd.run_cli('hello-world state refresh_cache')
except Exception as e:
if "failed: 409 Conflict" in e.args[0]:
return True
return False
spin.time_wait_noisy(lambda: check_cache_refresh_fails_409conflict(),
timeout_seconds=120.)
config = marathon.get_config(PACKAGE_NAME)
cpus = float(config['env']['HELLO_CPUS'])
del config['env']['DISABLE_STATE_CACHE']
cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config)
tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids)
check_running()
# caching reenabled, refresh_cache should succeed (eventually, once scheduler is up):
def check_cache_refresh():
return cmd.run_cli('hello-world state refresh_cache')
stdout = spin.time_wait_return(lambda: check_cache_refresh(),
timeout_seconds=120.)
assert "Received cmd: refresh" in stdout
def test_state_refresh_disable_cache():
'''Disables caching via a scheduler envvar'''
check_running()
task_ids = tasks.get_task_ids(PACKAGE_NAME, '')
# caching enabled by default:
stdout = cmd.run_cli('hello-world state refresh_cache')
assert "Received cmd: refresh" in stdout
config = marathon.get_config(PACKAGE_NAME)
cpus = float(config['env']['HELLO_CPUS'])
config['env']['DISABLE_STATE_CACHE'] = 'any-text-here'
cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config)
tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids)
check_running()
# caching disabled, refresh_cache should fail with a 409 error (eventually, once scheduler is up):
def check_cache_refresh_fails_409conflict():
try:
cmd.run_cli('hello-world state refresh_cache')
except Exception as e:
if "failed: 409 Conflict" in e.args[0]:
return True
return False
spin.time_wait_noisy(lambda: check_cache_refresh_fails_409conflict(), timeout_seconds=120.)
config = marathon.get_config(PACKAGE_NAME)
cpus = float(config['env']['HELLO_CPUS'])
del config['env']['DISABLE_STATE_CACHE']
cmd.request('put', marathon.api_url('apps/' + PACKAGE_NAME), json=config)
tasks.check_tasks_not_updated(PACKAGE_NAME, '', task_ids)
check_running()
# caching reenabled, refresh_cache should succeed (eventually, once scheduler is up):
def check_cache_refresh():
return cmd.run_cli('hello-world state refresh_cache')
stdout = spin.time_wait_return(lambda: check_cache_refresh(), timeout_seconds=120.)
assert "Received cmd: refresh" in stdout
def test_increase_cpu():
def plan_waiting():
try:
pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME))
if pl['status'] == 'WAITING':
return True
except:
pass
return False
def plan_complete():
try:
pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME))
if pl['status'] == 'COMPLETE':
return True
except:
pass
return False
config = marathon.get_config(SERVICE_NAME)
config['env']['BROKER_CPUS'] = str(0.1 + float(config['env']['BROKER_CPUS']))
marathon.update_app(SERVICE_NAME, config)
spin.time_wait_return(plan_waiting)
pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME))
assert pl['status'] == 'WAITING'
assert pl['phases'][0]['status'] == 'WAITING'
assert pl['phases'][0]['steps'][0]['status'] == 'WAITING'
assert pl['phases'][0]['steps'][1]['status'] == 'WAITING'
for step in range (2, DEFAULT_BROKER_COUNT +1 ):
assert pl['phases'][0]['steps'][step]['status'] == 'PENDING'
# all tasks are still running
tasks.check_running(SERVICE_NAME, DEFAULT_BROKER_COUNT + 1)
broker_ids = tasks.get_task_ids(SERVICE_NAME, '{}-0-{}'.format(DEFAULT_POD_TYPE, DEFAULT_TASK_NAME))
service_cli('plan continue {} {}'.format(DEFAULT_PLAN_NAME, DEFAULT_PHASE_NAME))
tasks.check_tasks_updated(SERVICE_NAME, '{}-0-{}'.format(DEFAULT_POD_TYPE, DEFAULT_TASK_NAME), broker_ids)
tasks.check_running(SERVICE_NAME, DEFAULT_BROKER_COUNT + 1)
pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME))
assert pl['status'] == 'WAITING'
assert pl['phases'][0]['status'] == 'WAITING'
assert pl['phases'][0]['steps'][0]['status'] == 'COMPLETE'
assert pl['phases'][0]['steps'][1]['status'] == 'WAITING'
for step in range(2, DEFAULT_BROKER_COUNT + 1):
assert pl['phases'][0]['steps'][step]['status'] == 'PENDING'
broker_ids = tasks.get_task_ids(SERVICE_NAME, '{}-1-{}'.format(DEFAULT_POD_TYPE, DEFAULT_TASK_NAME))
service_cli('plan continue {} {}'.format(DEFAULT_PLAN_NAME, DEFAULT_PHASE_NAME))
tasks.check_tasks_updated(SERVICE_NAME, '{}-1-{}'.format(DEFAULT_POD_TYPE, DEFAULT_TASK_NAME), broker_ids)
spin.time_wait_return(plan_complete)
pl = service_cli('plan show {}'.format(DEFAULT_PLAN_NAME))
assert pl['status'] == 'COMPLETE'
assert pl['phases'][0]['status'] == 'COMPLETE'
for step in range(DEFAULT_BROKER_COUNT + 1):
assert pl['phases'][0]['steps'][step]['status'] == 'COMPLETE'
broker_count_check(DEFAULT_BROKER_COUNT + 1)
def get_plan(service_name, plan):
def fn():
return sdk_api.get(service_name, "/v1/plans/{}".format(plan))
return sdk_spin.time_wait_return(fn)
def wait_for_completed_deployment(service_name):
def fn():
return deployment_plan_is_finished(service_name)
return sdk_spin.time_wait_return(fn)
def wait_for_completed_recovery(service_name):
def fn():
return recovery_plan_is_finished(service_name)
return sdk_spin.time_wait_return(fn)
