def bash_openstack(connection,
choice_spec,
*pargs,
**kwargs):
"""
Workload driver. Executes a workload as specified by the `choice_spec`
parameter against a nova-cluster.
`connection` : Connection object returned by call to kong.nova.API
`choice_spec` : list of BasherChoice actions to run on the cluster
`pargs` : additional positional arguments to bash_openstack
`kargs` : keyword arguments to the constructor of `test_case`
`duration` = how long this test should last (3 sec)
`sleep_time` = time to sleep between actions (in msec)
`test_name` = human readable workload description
(default: unnamed test)
`max_vms` = maximum number of machines to launch
(default: 32)
`seed` = random seed (default: None)
"""
# get keyword arguments
duration=kwargs.get('duration', datetime.timedelta(seconds=10))
seed=kwargs.get('seed', None)
sleep_time=float(kwargs.get('sleep_time', 3000))/1000
max_vms=int(kwargs.get('max_vms', 32))
test_name=kwargs.get('test_name', 'unamed test')
computes = _get_compute_nodes()
utils.util.execute_on_all(computes, "rm -f /var/log/nova/*.log")
random.seed(seed)
cases = _create_cases(choice_spec)
test_end_time = time.time() + duration.seconds
state = State(max_vms=max_vms)
retry_list = []
last_retry = time.time()
cooldown = False
logcheck_count = 0
test_succeeded = True
logging.debug('=== Test \"%s\" on %s ===' %
(test_name, time.asctime(time.localtime())))
for kw in kwargs:
logging.debug('\t%s = %s', kw, kwargs[kw])
while True:
if not cooldown:
if time.time() < test_end_time:
case = random.choice(cases)
retry = case.invoke(connection, state)
if retry != None:
retry_list.append(retry)
else:
logging.info('Cooling down...')
cooldown = True
if cooldown and len(retry_list) == 0:
if _error_in_logs(computes):
test_succeeded = False
break
# Retry verifications every 5 seconds.
if time.time() - last_retry > 5:
logging.debug('retry verifications for %d tasks', len(retry_list))
new_retry_list = []
for v in retry_list:
if not v.retry():
new_retry_list.append(v)
retry_list = new_retry_list
last_retry = time.time()
time.sleep(sleep_time)
# Check error logs after 100 actions
if logcheck_count > 100:
if _error_in_logs(computes):
test_succeeded = False
break
else:
logcheck_count = 0
else:
logcheck_count = logcheck_count + 1
# Cleanup
logging.info('Cleaning up: terminating virtual machines...')
vms = state.get_machines()
active_vms = [v for k, v in vms.iteritems() if v and v[1] == 'ACTIVE']
for target in active_vms:
kill_target = target[0]
connection.delete_server(kill_target['id'])
# check to see that the server was actually killed
try:
url = '/servers/%s' % kill_target['id']
connection.poll_request_status('GET', url, 404, timeout=60)
except TimeoutException:
raise
logging.info('killed %s' % kill_target['id'])
state.set_machine_state(kill_target['id'], None)
if test_succeeded:
logging.info('*** Test succeeded ***')
else:
logging.info('*** Test had errors ***')
return test_succeeded
