def check_status():
# python-novaclient has resources available to its client
# that all implement a get() method taking an identifier
# for the singular resource to retrieve.
try:
thing = things.get(thing_id)
except not_found_exception:
if allow_notfound:
return True
else:
raise
new_status = thing.status
# Some components are reporting error status in lower case
# so case sensitive comparisons can really mess things
# up.
if new_status.lower() == error_status.lower():
message = ("%s failed to get to expected status (%s). "
"In %s state.") % (thing, expected_status,
new_status)
raise exceptions.BuildErrorException(message,
server_id=thing_id)
elif new_status == expected_status and expected_status is not None:
return True # All good.
LOG.debug(
"Waiting for %s to get to %s status. "
"Currently in %s status", thing, log_status, new_status)
def create_subnet(cls, network, gateway=None, cidr=None, mask_bits=None,
**kwargs):
"""Wrapper utility that returns a test subnet."""
# The cidr and mask_bits depend on the ip version.
if cls._ip_version == 4:
cidr = cidr or netaddr.IPNetwork(CONF.network.tenant_network_cidr)
mask_bits = mask_bits or CONF.network.tenant_network_mask_bits
elif cls._ip_version == 6:
cidr = (
cidr or netaddr.IPNetwork(CONF.network.tenant_network_v6_cidr))
mask_bits = mask_bits or CONF.network.tenant_network_v6_mask_bits
# Find a cidr that is not in use yet and create a subnet with it
for subnet_cidr in cidr.subnet(mask_bits):
if not gateway:
gateway = str(netaddr.IPAddress(subnet_cidr) + 1)
try:
resp, body = cls.client.create_subnet(
network_id=network['id'],
cidr=str(subnet_cidr),
ip_version=cls._ip_version,
gateway_ip=gateway,
**kwargs)
break
except exceptions.BadRequest as e:
is_overlapping_cidr = 'overlaps with another subnet' in str(e)
# Unset gateway value if there is an overlapping subnet
gateway = None
if not is_overlapping_cidr:
raise
else:
message = 'Available CIDR for subnet creation could not be found'
raise exceptions.BuildErrorException(message)
subnet = body['subnet']
cls.subnets.append(subnet)
return subnet
def create_subnet(cls, network):
"""Wrapper utility that returns a test subnet."""
# The cidr and mask_bits depend on the ip version.
if cls._ip_version == 4:
cidr = netaddr.IPNetwork(CONF.network.tenant_network_cidr)
mask_bits = CONF.network.tenant_network_mask_bits
elif cls._ip_version == 6:
cidr = netaddr.IPNetwork(CONF.network.tenant_network_v6_cidr)
mask_bits = CONF.network.tenant_network_v6_mask_bits
# Find a cidr that is not in use yet and create a subnet with it
for subnet_cidr in cidr.subnet(mask_bits):
try:
resp, body = cls.client.create_subnet(
network_id=network['id'],
cidr=str(subnet_cidr),
ip_version=cls._ip_version)
break
except exceptions.BadRequest as e:
is_overlapping_cidr = 'overlaps with another subnet' in str(e)
if not is_overlapping_cidr:
raise
else:
message = 'Available CIDR for subnet creation could not be found'
raise exceptions.BuildErrorException(message)
subnet = body['subnet']
cls.subnets.append(subnet)
return subnet
def _create_subnet(self, subnet_name, tenant_id, network_id):
if not self.tempest_client:
body = {
'subnet': {
'name': subnet_name,
'tenant_id': tenant_id,
'network_id': network_id,
'ip_version': 4
}
}
base_cidr = netaddr.IPNetwork(self.config.network.tenant_network_cidr)
mask_bits = self.config.network.tenant_network_mask_bits
for subnet_cidr in base_cidr.subnet(mask_bits):
try:
if self.tempest_client:
resp, resp_body = self.network_admin_client.create_subnet(
network_id,
str(subnet_cidr),
name=subnet_name,
tenant_id=tenant_id)
else:
body['subnet']['cidr'] = str(subnet_cidr)
resp_body = self.network_admin_client.create_subnet(body)
break
except exceptions.BadRequest as e:
if 'overlaps with another subnet' not in str(e):
raise
else:
e = exceptions.BuildErrorException()
e.message = 'Available CIDR for subnet creation could not be found'
raise e
return resp_body['subnet']
def _create_subnet(self, subnet_name, tenant_id, network_id):
base_cidr = netaddr.IPNetwork(CONF.network.tenant_network_cidr)
mask_bits = CONF.network.tenant_network_mask_bits
for subnet_cidr in base_cidr.subnet(mask_bits):
try:
if self.network_resources:
_, resp_body = self.network_admin_client.\
create_subnet(
network_id=network_id, cidr=str(subnet_cidr),
name=subnet_name,
tenant_id=tenant_id,
enable_dhcp=self.network_resources['dhcp'],
ip_version=4)
else:
_, resp_body = self.network_admin_client.\
create_subnet(network_id=network_id,
cidr=str(subnet_cidr),
name=subnet_name,
tenant_id=tenant_id,
ip_version=4)
break
except exceptions.BadRequest as e:
if 'overlaps with another subnet' not in str(e):
raise
else:
e = exceptions.BuildErrorException()
e.message = 'Available CIDR for subnet creation could not be found'
raise e
return resp_body['subnet']
def _create_subnet(self, network):
# The cidr and mask_bits depend on the ip version.
if self._ip_version == 4:
cidr = netaddr.IPNetwork(CONF.network.project_network_cidr
or "192.168.101.0/24")
mask_bits = CONF.network.project_network_mask_bits or 24
elif self._ip_version == 6:
cidr = netaddr.IPNetwork(CONF.network.project_network_v6_cidr)
mask_bits = CONF.network.project_network_v6_mask_bits
# Find a cidr that is not in use yet and create a subnet with it
for subnet_cidr in cidr.subnet(mask_bits):
try:
body = self.admin_subnet_client.create_subnet(
network_id=network['id'],
cidr=str(subnet_cidr),
ip_version=self._ip_version)
break
except exceptions.BadRequest as e:
is_overlapping_cidr = 'overlaps with another subnet' in str(e)
if not is_overlapping_cidr:
raise
else:
message = ('Available CIDR for subnet creation '
'could not be found')
raise exceptions.BuildErrorException(message)
return body['subnet']
def wait_for_server_status(client, server_id, status, ready_wait=True):
"""Waits for a server to reach a given status."""
def _get_task_state(body):
task_state = body.get('OS-EXT-STS:task_state', None)
return task_state
# NOTE(afazekas): UNKNOWN status possible on ERROR
# or in a very early stage.
resp, body = client.get_server(server_id)
old_status = server_status = body['status']
old_task_state = task_state = _get_task_state(body)
start_time = int(time.time())
while True:
# NOTE(afazekas): Now the BUILD status only reached
# between the UNKOWN->ACTIVE transition.
# TODO(afazekas): enumerate and validate the stable status set
if status == 'BUILD' and server_status != 'UNKNOWN':
return
if server_status == status:
if ready_wait:
if status == 'BUILD':
return
# NOTE(afazekas): The instance is in "ready for action state"
# when no task in progress
# NOTE(afazekas): Converted to string bacuse of the XML
# responses
if str(task_state) == "None":
# without state api extension 3 sec usually enough
time.sleep(CONFIG.compute.ready_wait)
return
else:
return
time.sleep(client.build_interval)
resp, body = client.get_server(server_id)
server_status = body['status']
task_state = _get_task_state(body)
if (server_status != old_status) or (task_state != old_task_state):
LOG.info('State transition "%s" ==> "%s" after %d second wait',
'/'.join((old_status, str(old_task_state))), '/'.join(
(server_status, str(task_state))),
time.time() - start_time)
if server_status == 'ERROR':
raise exceptions.BuildErrorException(server_id=server_id)
timed_out = int(time.time()) - start_time >= client.build_timeout
if timed_out:
message = ('Server %s failed to reach %s status within the '
'required time (%s s).' %
(server_id, status, client.build_timeout))
message += ' Current status: %s.' % server_status
raise exceptions.TimeoutException(message)
old_status = server_status
old_task_state = task_state
def check_cluster_active(self, cluster_id):
timeout = CONF.data_processing.cluster_timeout
s_time = timeutils.utcnow()
while timeutils.delta_seconds(s_time, timeutils.utcnow()) < timeout:
cluster = self.client.clusters.get(cluster_id)
if cluster.status == 'Active':
return
if cluster.status == 'Error':
raise exceptions.BuildErrorException(
'Cluster failed to build and is in "Error" status.')
time.sleep(CONF.data_processing.request_timeout)
raise exceptions.TimeoutException(
'Cluster failed to get to "Active status within %d seconds.' %
timeout)
def create_subnet(cls,
network,
gateway='',
cidr=None,
mask_bits=None,
ip_version=None,
client=None,
**kwargs):
"""Wrapper utility that returns a test subnet."""
# allow tests to use admin client
if not client:
client = cls.subnets_client
# The cidr and mask_bits depend on the ip version.
ip_version = ip_version if ip_version is not None else cls._ip_version
gateway_not_set = gateway == ''
if ip_version == 4:
cidr = cidr or netaddr.IPNetwork(CONF.network.project_network_cidr)
mask_bits = mask_bits or CONF.network.project_network_mask_bits
elif ip_version == 6:
cidr = (cidr
or netaddr.IPNetwork(CONF.network.project_network_v6_cidr))
mask_bits = mask_bits or CONF.network.project_network_v6_mask_bits
# Find a cidr that is not in use yet and create a subnet with it
for subnet_cidr in cidr.subnet(mask_bits):
if gateway_not_set:
gateway_ip = str(netaddr.IPAddress(subnet_cidr) + 1)
else:
gateway_ip = gateway
try:
body = client.create_subnet(network_id=network['id'],
cidr=str(subnet_cidr),
ip_version=ip_version,
gateway_ip=gateway_ip,
**kwargs)
break
except lib_exc.BadRequest as e:
is_overlapping_cidr = 'overlaps with another subnet' in str(e)
if not is_overlapping_cidr:
raise
else:
message = 'Available CIDR for subnet creation could not be found'
raise exceptions.BuildErrorException(message)
subnet = body['subnet']
cls.addClassResourceCleanup(test_utils.call_and_ignore_notfound_exc,
cls.subnets_client.delete_subnet,
subnet['id'])
cls.subnets.append(subnet)
return subnet
def check_cluster_active(self, cluster_id):
timeout = TEMPEST_CONF.data_processing.cluster_timeout
s_time = timeutils.utcnow()
while timeutils.delta_seconds(s_time, timeutils.utcnow()) < timeout:
cluster = self.client.clusters.get(cluster_id)
if cluster.status == CLUSTER_STATUS_ACTIVE:
return
if cluster.status == CLUSTER_STATUS_ERROR:
raise exceptions.BuildErrorException(
'Cluster failed to build and is in %s status.' %
CLUSTER_STATUS_ERROR)
time.sleep(TEMPEST_CONF.data_processing.request_timeout)
raise exceptions.TimeoutException(
'Cluster failed to get to %s status within %d seconds.' %
(CLUSTER_STATUS_ACTIVE, timeout))
def wait_for_server_termination(self, server_id, ignore_error=False):
"""Waits for server to reach termination."""
start_time = int(time.time())
while True:
try:
resp, body = self.get_server(server_id)
except exceptions.NotFound:
return
server_status = body['status']
if server_status == 'ERROR' and not ignore_error:
raise exceptions.BuildErrorException(server_id=server_id)
if int(time.time()) - start_time >= self.build_timeout:
raise exceptions.TimeoutException
time.sleep(self.build_interval)
def wait_for_server_termination(client, server_id, ignore_error=False):
"""Waits for server to reach termination."""
start_time = int(time.time())
while True:
try:
body = client.show_server(server_id)['server']
except lib_exc.NotFound:
return
server_status = body['status']
if server_status == 'ERROR' and not ignore_error:
raise exceptions.BuildErrorException(server_id=server_id)
if int(time.time()) - start_time >= client.build_timeout:
raise exceptions.TimeoutException
time.sleep(client.build_interval)
def wait_for_server_status(self, server_id, status):
"""Waits for a server to reach a given status"""
resp, body = self.get_server(server_id)
server_status = body['status']
start = int(time.time())
while (server_status != status):
time.sleep(self.build_interval)
resp, body = self.get_server(server_id)
server_status = body['status']
if (server_status == 'ERROR'):
message = 'Server %s entered ERROR status.' % server_id
raise exceptions.BuildErrorException(message)
if (int(time.time()) - start >= self.build_timeout):
message = 'Server %s failed to reach ACTIVE status within the \
required time (%s s).' % (server_id, self.build_timeout)
raise exceptions.TimeoutException(message)
def wait_for_server_status(self, server_id, status):
"""Waits for a server to reach a given status."""
resp, body = self.get_server(server_id)
server_status = body['status']
start = int(time.time())
while (server_status != status):
time.sleep(self.build_interval)
resp, body = self.get_server(server_id)
server_status = body['status']
if server_status == 'ERROR':
raise exceptions.BuildErrorException(server_id=server_id)
timed_out = int(time.time()) - start >= self.build_timeout
if server_status != status and timed_out:
message = ('Server %s failed to reach %s status within the '
'required time (%s s).' %
(server_id, status, self.build_timeout))
message += ' Current status: %s.' % server_status
raise exceptions.TimeoutException(message)
def check_status():
# python-novaclient has resources available to its client
# that all implement a get() method taking an identifier
# for the singular resource to retrieve.
try:
thing = things.get(thing_id)
except not_found_exception:
if allow_notfound:
return True
else:
raise
new_status = thing.status
if new_status == error_status:
message = "%s failed to get to expected status. \
In %s state." % (thing, new_status)
raise exceptions.BuildErrorException(message)
elif new_status == expected_status and expected_status is not None:
return True # All good.
LOG.debug(
"Waiting for %s to get to %s status. "
"Currently in %s status", thing, log_status, new_status)
def wait_for_server_status(client,
server_id,
status,
ready_wait=True,
extra_timeout=0,
raise_on_error=True):
"""Waits for a server to reach a given status."""
def _get_task_state(body):
return body.get('OS-EXT-STS:task_state', None)
# NOTE(afazekas): UNKNOWN status possible on ERROR
# or in a very early stage.
body = client.get_server(server_id)
old_status = server_status = body['status']
old_task_state = task_state = _get_task_state(body)
start_time = int(time.time())
timeout = client.build_timeout + extra_timeout
while True:
# NOTE(afazekas): Now the BUILD status only reached
# between the UNKNOWN->ACTIVE transition.
# TODO(afazekas): enumerate and validate the stable status set
if status == 'BUILD' and server_status != 'UNKNOWN':
return
if server_status == status:
if ready_wait:
if status == 'BUILD':
return
# NOTE(afazekas): The instance is in "ready for action state"
# when no task in progress
# NOTE(afazekas): Converted to string bacuse of the XML
# responses
if str(task_state) == "None":
# without state api extension 3 sec usually enough
time.sleep(CONF.compute.ready_wait)
return
else:
return
time.sleep(client.build_interval)
body = client.get_server(server_id)
server_status = body['status']
task_state = _get_task_state(body)
if (server_status != old_status) or (task_state != old_task_state):
LOG.info('State transition "%s" ==> "%s" after %d second wait',
'/'.join((old_status, str(old_task_state))), '/'.join(
(server_status, str(task_state))),
time.time() - start_time)
if (server_status == 'ERROR') and raise_on_error:
if 'fault' in body:
raise exceptions.BuildErrorException(body['fault'],
server_id=server_id)
else:
raise exceptions.BuildErrorException(server_id=server_id)
timed_out = int(time.time()) - start_time >= timeout
if timed_out:
expected_task_state = 'None' if ready_wait else 'n/a'
message = ('Server %(server_id)s failed to reach %(status)s '
'status and task state "%(expected_task_state)s" '
'within the required time (%(timeout)s s).' % {
'server_id': server_id,
'status': status,
'expected_task_state': expected_task_state,
'timeout': timeout
})
message += ' Current status: %s.' % server_status
message += ' Current task state: %s.' % task_state
caller = misc_utils.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
old_status = server_status
old_task_state = task_state