def wait_for_server_status_to_active(self, server_id, status): """Waits for a server to reach a given status.""" region = CONF.compute.region endpoint_type = CONF.compute.endpoint_type build_interval = CONF.compute.build_interval build_timeout = CONF.compute.build_timeout disable_ssl_cert = CONF.identity.disable_ssl_certificate_validation ca_certs = CONF.identity.ca_certificates_file rs_client = rest_client.RestClient(self.auth_provider, "compute", region, endpoint_type, build_interval, build_timeout, disable_ssl_cert, ca_certs) resp, body = rs_client.get("servers/%s" % str(server_id)) body = jsonutils.loads(body) server_status = body['server']['status'] start = int(time.time()) while server_status != status: time.sleep(build_interval) rs_client = rest_client.RestClient(self.auth_provider, "compute", region, endpoint_type, build_interval, build_timeout, disable_ssl_cert, ca_certs) resp, body = rs_client.get("servers/%s" % str(server_id)) body = jsonutils.loads(body) server_status = body['server']['status'] if int(time.time()) - start >= build_timeout: message = ('server %s failed to reach' ' %s status (current %s) ' 'within the required time (%s s).' % (server_id, status, server_status, build_timeout)) raise exceptions.TimeoutException(message)
def wait_for_image_status(client, image_id, status): """Waits for an image to reach a given status. The client should have a get_image(image_id) method to get the image. The client should also have build_interval and build_timeout attributes. """ image = client.get_image(image_id) start = int(time.time()) while image['status'] != status: time.sleep(client.build_interval) image = client.get_image(image_id) status_curr = image['status'] if status_curr == 'ERROR': raise exceptions.AddImageException(image_id=image_id) # check the status again to avoid a false negative where we hit # the timeout at the same time that the image reached the expected # status if status_curr == status: return if int(time.time()) - start >= client.build_timeout: message = ('Image %(image_id)s failed to reach %(status)s state' '(current state %(status_curr)s) ' 'within the required time (%(timeout)s s).' % {'image_id': image_id, 'status': status, 'status_curr': status_curr, 'timeout': client.build_timeout}) caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def wait_for_bm_node_status(client, node_id, attr, status): """Waits for a baremetal node attribute to reach given status. The client should have a show_node(node_uuid) method to get the node. """ _, node = client.show_node(node_id) start = int(time.time()) while node[attr] != status: time.sleep(client.build_interval) _, node = client.show_node(node_id) status_curr = node[attr] if status_curr == status: return if int(time.time()) - start >= client.build_timeout: message = ('Node %(node_id)s failed to reach %(attr)s=%(status)s ' 'within the required time (%(timeout)s s).' % {'node_id': node_id, 'attr': attr, 'status': status, 'timeout': client.build_timeout}) message += ' Current state of %s: %s.' % (attr, status_curr) caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def _http_request(self, url, method, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) kwargs['headers'].setdefault('User-Agent', USER_AGENT) self._log_request(method, url, kwargs['headers']) conn = self.get_connection() try: url_parts = urlparse.urlparse(url) conn_url = posixpath.normpath(url_parts.path) LOG.debug('Actual Path: {path}'.format(path=conn_url)) if kwargs['headers'].get('Transfer-Encoding') == 'chunked': conn.putrequest(method, conn_url) for header, value in kwargs['headers'].items(): conn.putheader(header, value) conn.endheaders() chunk = kwargs['body'].read(CHUNKSIZE) # Chunk it, baby... while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = kwargs['body'].read(CHUNKSIZE) conn.send('0\r\n\r\n') else: conn.request(method, conn_url, **kwargs) resp = conn.getresponse() except socket.gaierror as e: message = ("Error finding address for %(url)s: %(e)s" % { 'url': url, 'e': e }) raise exc.EndpointNotFound(message) except (socket.error, socket.timeout) as e: message = ("Error communicating with %(endpoint)s %(e)s" % { 'endpoint': self.endpoint, 'e': e }) raise exc.TimeoutException(message) body_iter = ResponseBodyIterator(resp) # Read body into string if it isn't obviously image data if resp.getheader('content-type', None) != 'application/octet-stream': body_str = ''.join([body_chunk for body_chunk in body_iter]) body_iter = StringIO.StringIO(body_str) self._log_response(resp, None) else: self._log_response(resp, body_iter) return resp, body_iter
def _wait_until_ready(self, fw_id): target_states = ('ACTIVE', 'CREATED') def _wait(): firewall = self.client.show_firewall(fw_id) firewall = firewall['firewall'] return firewall['status'] in target_states if not test.call_until_true(_wait, CONF.network.build_timeout, CONF.network.build_interval): m = ("Timed out waiting for firewall %s to reach %s state(s)" % (fw_id, target_states)) raise exceptions.TimeoutException(m)
def exec_command(self, cmd): """ Execute the specified command on the server. Note that this method is reading whole command outputs to memory, thus shouldn't be used for large outputs. :returns: data read from standard output of the command. :raises: SSHExecCommandFailed if command returns nonzero status. The exception contains command status stderr content. """ ssh = self._get_ssh_connection() transport = ssh.get_transport() channel = transport.open_session() channel.fileno() # Register event pipe channel.exec_command(cmd) channel.shutdown_write() out_data = [] err_data = [] poll = select.poll() poll.register(channel, select.POLLIN) start_time = time.time() while True: ready = poll.poll(self.channel_timeout) if not any(ready): if not self._is_timed_out(start_time): continue raise exceptions.TimeoutException( "Command: '{0}' executed on host '{1}'.".format( cmd, self.host)) if not ready[0]: # If there is nothing to read. continue out_chunk = err_chunk = None if channel.recv_ready(): out_chunk = channel.recv(self.buf_size) out_data += out_chunk, if channel.recv_stderr_ready(): err_chunk = channel.recv_stderr(self.buf_size) err_data += err_chunk, if channel.closed and not err_chunk and not out_chunk: break exit_status = channel.recv_exit_status() if 0 != exit_status: raise exceptions.SSHExecCommandFailed(command=cmd, exit_status=exit_status, strerror=''.join(err_data)) return ''.join(out_data)
def _wait_for_floating_ip_status(self, floating_ip_id, status): """Waits for a floating_ip to reach a given status.""" build_timeout = CONF.compute.build_timeout build_interval = CONF.compute.build_interval floating_ip = self.admin_client.show_floatingip(floating_ip_id) shown_floating_ip = floating_ip['floatingip'] floating_ip_status = shown_floating_ip['status'] start = int(time.time()) while floating_ip_status != status: time.sleep(build_interval) floating_ip = self.admin_client.show_floatingip(floating_ip_id) shown_floating_ip = floating_ip['floatingip'] floating_ip_status = shown_floating_ip['status'] if int(time.time()) - start >= build_timeout: message = ( 'floating_ip %s failed to reach' ' %s status (current %s) ' 'within the required time (%s s).' % (floating_ip, status, floating_ip_status, build_timeout)) raise exceptions.TimeoutException(message)
def wait_for_resource_status(self, fetch, status, interval=None, timeout=None): """ @summary: Waits for a network resource to reach a status @param fetch: the callable to be used to query the resource status @type fecth: callable that takes no parameters and returns the resource @param status: the status that the resource has to reach @type status: String @param interval: the number of seconds to wait between each status query @type interval: Integer @param timeout: the maximum number of seconds to wait for the resource to reach the desired status @type timeout: Integer """ if not interval: interval = self.build_interval if not timeout: timeout = self.build_timeout start_time = time.time() while time.time() - start_time <= timeout: resource = fetch() if resource['status'] == status: return time.sleep(interval) # At this point, the wait has timed out message = 'Resource %s' % (str(resource)) message += ' failed to reach status %s' % status message += ' (current: %s)' % resource['status'] message += ' within the required time %s' % timeout caller = misc.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def wait_for_server_status(self, server_id, status, ready_wait=True, extra_timeout=0, raise_on_error=True): """Waits for a server to reach a given status.""" build_timeout = CONF.compute.build_timeout build_interval = CONF.boto.build_interval def _get_task_state(body): return body.get('OS-EXT-STS:task_state', None) rs = rest_client.RestClient(self.auth_provider, "compute", "RegionOne") resp, body = rs.get("servers/%s" % str(server_id)) body = json.loads(body) old_status = server_status = body['server']['status'] old_task_state = task_state = _get_task_state(body) start_time = int(time.time()) timeout = build_timeout + extra_timeout while True: if status == 'BUILD' and server_status != 'UNKNOWN': return if server_status == status: if ready_wait: if status == 'BUILD': return if str(task_state) == "None": time.sleep(CONF.compute.ready_wait) return else: return time.sleep(build_interval) resp, body = rs.get("servers/%s" % str(server_id)) body = json.loads(body) server_status = body['server']['status'] task_state = _get_task_state(body) if (server_status != old_status) or (task_state != old_task_state): oldstatus = '/'.join((old_status, str(old_task_state))) serverstatus = '/'.join((server_status, str(task_state))) waitsec = (time.time() - start_time) LOG.info( _LI('State transtion %(oldstatus)s => %(serverstatus)s' 'after %(waitsec)d second wait') % { 'oldstatus': oldstatus, 'serverstatus': serverstatus, 'waitsec': waitsec }) if (server_status == 'ERROR') and raise_on_error: if 'fault' in body: raise exceptions.BuildErrorException(body['fault'], server_id=server_id) else: raise exceptions.BuildErrorException(server_id=server_id) timed_out = int(time.time()) - start_time >= timeout if timed_out: expected_task_state = 'None' if ready_wait else 'n/a' message = ('Server %(server_id)s failed to reach %(status)s ' 'status and task state "%(expected_task_state)s" ' 'within the required time (%(timeout)s s).' % { 'server_id': server_id, 'status': status, 'expected_task_state': expected_task_state, 'timeout': timeout }) message += ' Current status: %s.' % server_status message += ' Current task state: %s.' % task_state caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message) old_status = server_status old_task_state = task_state
def wait_for_server_status(client, server_id, status, ready_wait=True, extra_timeout=0, raise_on_error=True): """Waits for a server to reach a given status.""" def _get_task_state(body): return body.get('OS-EXT-STS:task_state', None) # NOTE(afazekas): UNKNOWN status possible on ERROR # or in a very early stage. body = client.get_server(server_id) old_status = server_status = body['status'] old_task_state = task_state = _get_task_state(body) start_time = int(time.time()) timeout = client.build_timeout + extra_timeout while True: # NOTE(afazekas): Now the BUILD status only reached # between the UNKNOWN->ACTIVE transition. # TODO(afazekas): enumerate and validate the stable status set if status == 'BUILD' and server_status != 'UNKNOWN': return if server_status == status: if ready_wait: if status == 'BUILD': return # NOTE(afazekas): The instance is in "ready for action state" # when no task in progress # NOTE(afazekas): Converted to string bacuse of the XML # responses if str(task_state) == "None": # without state api extension 3 sec usually enough time.sleep(CONF.compute.ready_wait) return else: return time.sleep(client.build_interval) body = client.get_server(server_id) server_status = body['status'] task_state = _get_task_state(body) if (server_status != old_status) or (task_state != old_task_state): LOG.info('State transition "%s" ==> "%s" after %d second wait', '/'.join((old_status, str(old_task_state))), '/'.join((server_status, str(task_state))), time.time() - start_time) if (server_status == 'ERROR') and raise_on_error: if 'fault' in body: raise exceptions.BuildErrorException(body['fault'], server_id=server_id) else: raise exceptions.BuildErrorException(server_id=server_id) timed_out = int(time.time()) - start_time >= timeout if timed_out: expected_task_state = 'None' if ready_wait else 'n/a' message = ('Server %(server_id)s failed to reach %(status)s ' 'status and task state "%(expected_task_state)s" ' 'within the required time (%(timeout)s s).' % {'server_id': server_id, 'status': status, 'expected_task_state': expected_task_state, 'timeout': timeout}) message += ' Current status: %s.' % server_status message += ' Current task state: %s.' % task_state caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message) old_status = server_status old_task_state = task_state