def wait_for_server_status_to_active(self, server_id, status):
"""Waits for a server to reach a given status."""
region = CONF.compute.region
endpoint_type = CONF.compute.endpoint_type
build_interval = CONF.compute.build_interval
build_timeout = CONF.compute.build_timeout
disable_ssl_cert = CONF.identity.disable_ssl_certificate_validation
ca_certs = CONF.identity.ca_certificates_file
rs_client = rest_client.RestClient(self.auth_provider, "compute",
region, endpoint_type,
build_interval, build_timeout,
disable_ssl_cert, ca_certs)
resp, body = rs_client.get("servers/%s" % str(server_id))
body = jsonutils.loads(body)
server_status = body['server']['status']
start = int(time.time())
while server_status != status:
time.sleep(build_interval)
rs_client = rest_client.RestClient(self.auth_provider, "compute",
region, endpoint_type,
build_interval, build_timeout,
disable_ssl_cert, ca_certs)
resp, body = rs_client.get("servers/%s" % str(server_id))
body = jsonutils.loads(body)
server_status = body['server']['status']
if int(time.time()) - start >= build_timeout:
message = ('server %s failed to reach'
' %s status (current %s) '
'within the required time (%s s).' %
(server_id, status, server_status, build_timeout))
raise exceptions.TimeoutException(message)
def wait_for_image_status(client, image_id, status):
"""Waits for an image to reach a given status.
The client should have a get_image(image_id) method to get the image.
The client should also have build_interval and build_timeout attributes.
"""
image = client.get_image(image_id)
start = int(time.time())
while image['status'] != status:
time.sleep(client.build_interval)
image = client.get_image(image_id)
status_curr = image['status']
if status_curr == 'ERROR':
raise exceptions.AddImageException(image_id=image_id)
# check the status again to avoid a false negative where we hit
# the timeout at the same time that the image reached the expected
# status
if status_curr == status:
return
if int(time.time()) - start >= client.build_timeout:
message = ('Image %(image_id)s failed to reach %(status)s state'
'(current state %(status_curr)s) '
'within the required time (%(timeout)s s).' %
{'image_id': image_id,
'status': status,
'status_curr': status_curr,
'timeout': client.build_timeout})
caller = misc_utils.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
def wait_for_bm_node_status(client, node_id, attr, status):
"""Waits for a baremetal node attribute to reach given status.
The client should have a show_node(node_uuid) method to get the node.
"""
_, node = client.show_node(node_id)
start = int(time.time())
while node[attr] != status:
time.sleep(client.build_interval)
_, node = client.show_node(node_id)
status_curr = node[attr]
if status_curr == status:
return
if int(time.time()) - start >= client.build_timeout:
message = ('Node %(node_id)s failed to reach %(attr)s=%(status)s '
'within the required time (%(timeout)s s).' %
{'node_id': node_id,
'attr': attr,
'status': status,
'timeout': client.build_timeout})
message += ' Current state of %s: %s.' % (attr, status_curr)
caller = misc_utils.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
def _http_request(self, url, method, **kwargs):
"""Send an http request with the specified characteristics.
Wrapper around httplib.HTTP(S)Connection.request to handle tasks such
as setting headers and error handling.
"""
# Copy the kwargs so we can reuse the original in case of redirects
kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {}))
kwargs['headers'].setdefault('User-Agent', USER_AGENT)
self._log_request(method, url, kwargs['headers'])
conn = self.get_connection()
try:
url_parts = urlparse.urlparse(url)
conn_url = posixpath.normpath(url_parts.path)
LOG.debug('Actual Path: {path}'.format(path=conn_url))
if kwargs['headers'].get('Transfer-Encoding') == 'chunked':
conn.putrequest(method, conn_url)
for header, value in kwargs['headers'].items():
conn.putheader(header, value)
conn.endheaders()
chunk = kwargs['body'].read(CHUNKSIZE)
# Chunk it, baby...
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = kwargs['body'].read(CHUNKSIZE)
conn.send('0\r\n\r\n')
else:
conn.request(method, conn_url, **kwargs)
resp = conn.getresponse()
except socket.gaierror as e:
message = ("Error finding address for %(url)s: %(e)s" % {
'url': url,
'e': e
})
raise exc.EndpointNotFound(message)
except (socket.error, socket.timeout) as e:
message = ("Error communicating with %(endpoint)s %(e)s" % {
'endpoint': self.endpoint,
'e': e
})
raise exc.TimeoutException(message)
body_iter = ResponseBodyIterator(resp)
# Read body into string if it isn't obviously image data
if resp.getheader('content-type', None) != 'application/octet-stream':
body_str = ''.join([body_chunk for body_chunk in body_iter])
body_iter = StringIO.StringIO(body_str)
self._log_response(resp, None)
else:
self._log_response(resp, body_iter)
return resp, body_iter
def _wait_until_ready(self, fw_id):
target_states = ('ACTIVE', 'CREATED')
def _wait():
firewall = self.client.show_firewall(fw_id)
firewall = firewall['firewall']
return firewall['status'] in target_states
if not test.call_until_true(_wait, CONF.network.build_timeout,
CONF.network.build_interval):
m = ("Timed out waiting for firewall %s to reach %s state(s)" %
(fw_id, target_states))
raise exceptions.TimeoutException(m)
def exec_command(self, cmd):
"""
Execute the specified command on the server.
Note that this method is reading whole command outputs to memory, thus
shouldn't be used for large outputs.
:returns: data read from standard output of the command.
:raises: SSHExecCommandFailed if command returns nonzero
status. The exception contains command status stderr content.
"""
ssh = self._get_ssh_connection()
transport = ssh.get_transport()
channel = transport.open_session()
channel.fileno() # Register event pipe
channel.exec_command(cmd)
channel.shutdown_write()
out_data = []
err_data = []
poll = select.poll()
poll.register(channel, select.POLLIN)
start_time = time.time()
while True:
ready = poll.poll(self.channel_timeout)
if not any(ready):
if not self._is_timed_out(start_time):
continue
raise exceptions.TimeoutException(
"Command: '{0}' executed on host '{1}'.".format(
cmd, self.host))
if not ready[0]: # If there is nothing to read.
continue
out_chunk = err_chunk = None
if channel.recv_ready():
out_chunk = channel.recv(self.buf_size)
out_data += out_chunk,
if channel.recv_stderr_ready():
err_chunk = channel.recv_stderr(self.buf_size)
err_data += err_chunk,
if channel.closed and not err_chunk and not out_chunk:
break
exit_status = channel.recv_exit_status()
if 0 != exit_status:
raise exceptions.SSHExecCommandFailed(command=cmd,
exit_status=exit_status,
strerror=''.join(err_data))
return ''.join(out_data)
def _wait_for_floating_ip_status(self, floating_ip_id, status):
"""Waits for a floating_ip to reach a given status."""
build_timeout = CONF.compute.build_timeout
build_interval = CONF.compute.build_interval
floating_ip = self.admin_client.show_floatingip(floating_ip_id)
shown_floating_ip = floating_ip['floatingip']
floating_ip_status = shown_floating_ip['status']
start = int(time.time())
while floating_ip_status != status:
time.sleep(build_interval)
floating_ip = self.admin_client.show_floatingip(floating_ip_id)
shown_floating_ip = floating_ip['floatingip']
floating_ip_status = shown_floating_ip['status']
if int(time.time()) - start >= build_timeout:
message = (
'floating_ip %s failed to reach'
' %s status (current %s) '
'within the required time (%s s).' %
(floating_ip, status, floating_ip_status, build_timeout))
raise exceptions.TimeoutException(message)
def wait_for_resource_status(self,
fetch,
status,
interval=None,
timeout=None):
"""
@summary: Waits for a network resource to reach a status
@param fetch: the callable to be used to query the resource status
@type fecth: callable that takes no parameters and returns the resource
@param status: the status that the resource has to reach
@type status: String
@param interval: the number of seconds to wait between each status
query
@type interval: Integer
@param timeout: the maximum number of seconds to wait for the resource
to reach the desired status
@type timeout: Integer
"""
if not interval:
interval = self.build_interval
if not timeout:
timeout = self.build_timeout
start_time = time.time()
while time.time() - start_time <= timeout:
resource = fetch()
if resource['status'] == status:
return
time.sleep(interval)
# At this point, the wait has timed out
message = 'Resource %s' % (str(resource))
message += ' failed to reach status %s' % status
message += ' (current: %s)' % resource['status']
message += ' within the required time %s' % timeout
caller = misc.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
def wait_for_server_status(self,
server_id,
status,
ready_wait=True,
extra_timeout=0,
raise_on_error=True):
"""Waits for a server to reach a given status."""
build_timeout = CONF.compute.build_timeout
build_interval = CONF.boto.build_interval
def _get_task_state(body):
return body.get('OS-EXT-STS:task_state', None)
rs = rest_client.RestClient(self.auth_provider, "compute", "RegionOne")
resp, body = rs.get("servers/%s" % str(server_id))
body = json.loads(body)
old_status = server_status = body['server']['status']
old_task_state = task_state = _get_task_state(body)
start_time = int(time.time())
timeout = build_timeout + extra_timeout
while True:
if status == 'BUILD' and server_status != 'UNKNOWN':
return
if server_status == status:
if ready_wait:
if status == 'BUILD':
return
if str(task_state) == "None":
time.sleep(CONF.compute.ready_wait)
return
else:
return
time.sleep(build_interval)
resp, body = rs.get("servers/%s" % str(server_id))
body = json.loads(body)
server_status = body['server']['status']
task_state = _get_task_state(body)
if (server_status != old_status) or (task_state != old_task_state):
oldstatus = '/'.join((old_status, str(old_task_state)))
serverstatus = '/'.join((server_status, str(task_state)))
waitsec = (time.time() - start_time)
LOG.info(
_LI('State transtion %(oldstatus)s => %(serverstatus)s'
'after %(waitsec)d second wait') % {
'oldstatus': oldstatus,
'serverstatus': serverstatus,
'waitsec': waitsec
})
if (server_status == 'ERROR') and raise_on_error:
if 'fault' in body:
raise exceptions.BuildErrorException(body['fault'],
server_id=server_id)
else:
raise exceptions.BuildErrorException(server_id=server_id)
timed_out = int(time.time()) - start_time >= timeout
if timed_out:
expected_task_state = 'None' if ready_wait else 'n/a'
message = ('Server %(server_id)s failed to reach %(status)s '
'status and task state "%(expected_task_state)s" '
'within the required time (%(timeout)s s).' % {
'server_id': server_id,
'status': status,
'expected_task_state': expected_task_state,
'timeout': timeout
})
message += ' Current status: %s.' % server_status
message += ' Current task state: %s.' % task_state
caller = misc_utils.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
old_status = server_status
old_task_state = task_state
def wait_for_server_status(client, server_id, status, ready_wait=True,
extra_timeout=0, raise_on_error=True):
"""Waits for a server to reach a given status."""
def _get_task_state(body):
return body.get('OS-EXT-STS:task_state', None)
# NOTE(afazekas): UNKNOWN status possible on ERROR
# or in a very early stage.
body = client.get_server(server_id)
old_status = server_status = body['status']
old_task_state = task_state = _get_task_state(body)
start_time = int(time.time())
timeout = client.build_timeout + extra_timeout
while True:
# NOTE(afazekas): Now the BUILD status only reached
# between the UNKNOWN->ACTIVE transition.
# TODO(afazekas): enumerate and validate the stable status set
if status == 'BUILD' and server_status != 'UNKNOWN':
return
if server_status == status:
if ready_wait:
if status == 'BUILD':
return
# NOTE(afazekas): The instance is in "ready for action state"
# when no task in progress
# NOTE(afazekas): Converted to string bacuse of the XML
# responses
if str(task_state) == "None":
# without state api extension 3 sec usually enough
time.sleep(CONF.compute.ready_wait)
return
else:
return
time.sleep(client.build_interval)
body = client.get_server(server_id)
server_status = body['status']
task_state = _get_task_state(body)
if (server_status != old_status) or (task_state != old_task_state):
LOG.info('State transition "%s" ==> "%s" after %d second wait',
'/'.join((old_status, str(old_task_state))),
'/'.join((server_status, str(task_state))),
time.time() - start_time)
if (server_status == 'ERROR') and raise_on_error:
if 'fault' in body:
raise exceptions.BuildErrorException(body['fault'],
server_id=server_id)
else:
raise exceptions.BuildErrorException(server_id=server_id)
timed_out = int(time.time()) - start_time >= timeout
if timed_out:
expected_task_state = 'None' if ready_wait else 'n/a'
message = ('Server %(server_id)s failed to reach %(status)s '
'status and task state "%(expected_task_state)s" '
'within the required time (%(timeout)s s).' %
{'server_id': server_id,
'status': status,
'expected_task_state': expected_task_state,
'timeout': timeout})
message += ' Current status: %s.' % server_status
message += ' Current task state: %s.' % task_state
caller = misc_utils.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
old_status = server_status
old_task_state = task_state
