def wait_for_image_status(self, image_id, status): """Waits for a Image to reach a given status.""" start_time = time.time() old_value = value = self._get_image_status(image_id) while True: dtime = time.time() - start_time time.sleep(self.build_interval) if value != old_value: LOG.info('Value transition from "%s" to "%s"' 'in %d second(s).', old_value, value, dtime) if value == status: return value if value == 'killed': raise exceptions.ImageKilledException(image_id=image_id, status=status) if dtime > self.build_timeout: message = ('Time Limit Exceeded! (%ds)' 'while waiting for %s, ' 'but we got %s.' % (self.build_timeout, status, value)) caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message) time.sleep(self.build_interval) old_value = value value = self._get_image_status(image_id)
def wait_for_stack_status(self, stack_identifier, status, failure_pattern='^.*_FAILED$'): """Waits for a Stack to reach a given status.""" start = int(time.time()) fail_regexp = re.compile(failure_pattern) while True: try: body = self.show_stack(stack_identifier) except lib_exc.NotFound: if status == 'DELETE_COMPLETE': return stack_name = body['stack_name'] stack_status = body['stack_status'] if stack_status == status: return body if fail_regexp.search(stack_status): raise exceptions.StackBuildErrorException( stack_identifier=stack_identifier, stack_status=stack_status, stack_status_reason=body['stack_status_reason']) if int(time.time()) - start >= self.build_timeout: message = ( 'Stack %s failed to reach %s status (current: %s) ' 'within the required time (%s s).' % (stack_name, status, stack_status, self.build_timeout)) raise exceptions.TimeoutException(message) time.sleep(self.build_interval)
def wait_for_bm_node_status(client, node_id, attr, status): """Waits for a baremetal node attribute to reach given status. The client should have a show_node(node_uuid) method to get the node. """ _, node = client.show_node(node_id) start = int(time.time()) while node[attr] != status: time.sleep(client.build_interval) _, node = client.show_node(node_id) status_curr = node[attr] if status_curr == status: return if int(time.time()) - start >= client.build_timeout: message = ('Node %(node_id)s failed to reach %(attr)s=%(status)s ' 'within the required time (%(timeout)s s).' % { 'node_id': node_id, 'attr': attr, 'status': status, 'timeout': client.build_timeout }) message += ' Current state of %s: %s.' % (attr, status_curr) caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def wait_for_resource_status(self, stack_identifier, resource_name, status, failure_pattern='^.*_FAILED$'): """Waits for a Resource to reach a given status.""" start = int(time.time()) fail_regexp = re.compile(failure_pattern) while True: try: body = self.show_resource(stack_identifier, resource_name) except lib_exc.NotFound: # ignore this, as the resource may not have # been created yet pass else: resource_name = body['resource_name'] resource_status = body['resource_status'] if resource_status == status: return if fail_regexp.search(resource_status): raise exceptions.StackResourceBuildErrorException( resource_name=resource_name, stack_identifier=stack_identifier, resource_status=resource_status, resource_status_reason=body['resource_status_reason']) if int(time.time()) - start >= self.build_timeout: message = ('Resource %s failed to reach %s status ' '(current %s) within the required time (%s s).' % (resource_name, status, resource_status, self.build_timeout)) raise exceptions.TimeoutException(message) time.sleep(self.build_interval)
def wait_for_image_status(client, image_id, status): """Waits for an image to reach a given status. The client should have a get_image(image_id) method to get the image. The client should also have build_interval and build_timeout attributes. """ image = client.get_image(image_id) start = int(time.time()) while image['status'] != status: time.sleep(client.build_interval) image = client.get_image(image_id) status_curr = image['status'] if status_curr == 'ERROR': raise exceptions.AddImageException(image_id=image_id) # check the status again to avoid a false negative where we hit # the timeout at the same time that the image reached the expected # status if status_curr == status: return if int(time.time()) - start >= client.build_timeout: message = ('Image %(image_id)s failed to reach %(status)s state' '(current state %(status_curr)s) ' 'within the required time (%(timeout)s s).' % { 'image_id': image_id, 'status': status, 'status_curr': status_curr, 'timeout': client.build_timeout }) caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def _http_request(self, url, method, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) kwargs['headers'].setdefault('User-Agent', USER_AGENT) self._log_request(method, url, kwargs['headers']) conn = self.get_connection() try: url_parts = urlparse.urlparse(url) conn_url = posixpath.normpath(url_parts.path) LOG.debug('Actual Path: {path}'.format(path=conn_url)) if kwargs['headers'].get('Transfer-Encoding') == 'chunked': conn.putrequest(method, conn_url) for header, value in kwargs['headers'].items(): conn.putheader(header, value) conn.endheaders() chunk = kwargs['body'].read(CHUNKSIZE) # Chunk it, baby... while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = kwargs['body'].read(CHUNKSIZE) conn.send('0\r\n\r\n') else: conn.request(method, conn_url, **kwargs) resp = conn.getresponse() except socket.gaierror as e: message = ("Error finding address for %(url)s: %(e)s" % {'url': url, 'e': e}) raise exc.EndpointNotFound(message) except (socket.error, socket.timeout) as e: message = ("Error communicating with %(endpoint)s %(e)s" % {'endpoint': self.endpoint, 'e': e}) raise exc.TimeoutException(message) body_iter = ResponseBodyIterator(resp) # Read body into string if it isn't obviously image data if resp.getheader('content-type', None) != 'application/octet-stream': body_str = ''.join([body_chunk for body_chunk in body_iter]) body_iter = six.StringIO(body_str) self._log_response(resp, None) else: self._log_response(resp, body_iter) return resp, body_iter
def exec_command(self, cmd): """ Execute the specified command on the server. Note that this method is reading whole command outputs to memory, thus shouldn't be used for large outputs. :returns: data read from standard output of the command. :raises: SSHExecCommandFailed if command returns nonzero status. The exception contains command status stderr content. """ ssh = self._get_ssh_connection() transport = ssh.get_transport() channel = transport.open_session() channel.fileno() # Register event pipe channel.exec_command(cmd) channel.shutdown_write() out_data = [] err_data = [] poll = select.poll() poll.register(channel, select.POLLIN) start_time = time.time() while True: ready = poll.poll(self.channel_timeout) if not any(ready): if not self._is_timed_out(start_time): continue raise exceptions.TimeoutException( "Command: '{0}' executed on host '{1}'.".format( cmd, self.host)) if not ready[0]: # If there is nothing to read. continue out_chunk = err_chunk = None if channel.recv_ready(): out_chunk = channel.recv(self.buf_size) out_data += out_chunk, if channel.recv_stderr_ready(): err_chunk = channel.recv_stderr(self.buf_size) err_data += err_chunk, if channel.closed and not err_chunk and not out_chunk: break exit_status = channel.recv_exit_status() if 0 != exit_status: raise exceptions.SSHExecCommandFailed(command=cmd, exit_status=exit_status, strerror=''.join(err_data)) return ''.join(out_data)
def _check_connection(self, check_ip, port=80): def try_connect(check_ip, port): try: resp = urllib2.urlopen("http://{0}:{1}/".format( check_ip, port)) if resp.getcode() == 200: return True return False except IOError: return False except error.HTTPError: return False timeout = config.compute.ping_timeout start = time.time() while not try_connect(check_ip, port): if (time.time() - start) > timeout: message = "Timed out trying to connect to %s" % check_ip raise exceptions.TimeoutException(message)
def wait_for_volume_status(self, volume_id, status): """Waits for a Volume to reach a given status.""" body = self.show_volume(volume_id) volume_status = body['status'] start = int(time.time()) while volume_status != status: time.sleep(self.build_interval) body = self.show_volume(volume_id) volume_status = body['status'] if volume_status == 'error': raise exceptions.VolumeBuildErrorException(volume_id=volume_id) if int(time.time()) - start >= self.build_timeout: message = ( 'Volume %s failed to reach %s status (current: %s) ' 'within the required time ' '(%s s).' % (volume_id, status, volume_status, self.build_timeout)) raise exceptions.TimeoutException(message)
def wait_for_interface_status(self, server, port_id, status): """Waits for a interface to reach a given status.""" body = self.show_interface(server, port_id) interface_status = body['port_state'] start = int(time.time()) while(interface_status != status): time.sleep(self.build_interval) body = self.show_interface(server, port_id) interface_status = body['port_state'] timed_out = int(time.time()) - start >= self.build_timeout if interface_status != status and timed_out: message = ('Interface %s failed to reach %s status ' '(current %s) within the required time (%s s).' % (port_id, status, interface_status, self.build_timeout)) raise exceptions.TimeoutException(message) return body
def wait_for_resource_status(self, fetch, status, interval=None, timeout=None): """ @summary: Waits for a network resource to reach a status @param fetch: the callable to be used to query the resource status @type fecth: callable that takes no parameters and returns the resource @param status: the status that the resource has to reach @type status: String @param interval: the number of seconds to wait between each status query @type interval: Integer @param timeout: the maximum number of seconds to wait for the resource to reach the desired status @type timeout: Integer """ if not interval: interval = self.build_interval if not timeout: timeout = self.build_timeout start_time = time.time() while time.time() - start_time <= timeout: resource = fetch() if resource['status'] == status: return time.sleep(interval) # At this point, the wait has timed out message = 'Resource %s' % (str(resource)) message += ' failed to reach status %s' % status message += ' (current: %s)' % resource['status'] message += ' within the required time %s' % timeout caller = misc.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def wait_for_snapshot_status(self, snapshot_id, status): """Waits for a Snapshot to reach a given status.""" start_time = time.time() old_value = value = self._get_snapshot_status(snapshot_id) while True: dtime = time.time() - start_time time.sleep(self.build_interval) if value != old_value: LOG.info( 'Value transition from "%s" to "%s"' 'in %d second(s).', old_value, value, dtime) if (value == status): return value if dtime > self.build_timeout: message = ('Time Limit Exceeded! (%ds)' 'while waiting for %s, ' 'but we got %s.' % (self.build_timeout, status, value)) raise exceptions.TimeoutException(message) time.sleep(self.build_interval) old_value = value value = self._get_snapshot_status(snapshot_id)
def wait_for_server_status(client, server_id, status, ready_wait=True, extra_timeout=0, raise_on_error=True): """Waits for a server to reach a given status.""" def _get_task_state(body): return body.get('OS-EXT-STS:task_state', None) # NOTE(afazekas): UNKNOWN status possible on ERROR # or in a very early stage. body = client.get_server(server_id) old_status = server_status = body['status'] old_task_state = task_state = _get_task_state(body) start_time = int(time.time()) timeout = client.build_timeout + extra_timeout while True: # NOTE(afazekas): Now the BUILD status only reached # between the UNKNOWN->ACTIVE transition. # TODO(afazekas): enumerate and validate the stable status set if status == 'BUILD' and server_status != 'UNKNOWN': return if server_status == status: if ready_wait: if status == 'BUILD': return # NOTE(afazekas): The instance is in "ready for action state" # when no task in progress # NOTE(afazekas): Converted to string because of the XML # responses if str(task_state) == "None": # without state api extension 3 sec usually enough time.sleep(CONF.compute.ready_wait) return else: return time.sleep(client.build_interval) body = client.get_server(server_id) server_status = body['status'] task_state = _get_task_state(body) if (server_status != old_status) or (task_state != old_task_state): LOG.info('State transition "%s" ==> "%s" after %d second wait', '/'.join((old_status, str(old_task_state))), '/'.join( (server_status, str(task_state))), time.time() - start_time) if (server_status == 'ERROR') and raise_on_error: if 'fault' in body: raise exceptions.BuildErrorException(body['fault'], server_id=server_id) else: raise exceptions.BuildErrorException(server_id=server_id) timed_out = int(time.time()) - start_time >= timeout if timed_out: expected_task_state = 'None' if ready_wait else 'n/a' message = ('Server %(server_id)s failed to reach %(status)s ' 'status and task state "%(expected_task_state)s" ' 'within the required time (%(timeout)s s).' % { 'server_id': server_id, 'status': status, 'expected_task_state': expected_task_state, 'timeout': timeout }) message += ' Current status: %s.' % server_status message += ' Current task state: %s.' % task_state caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message) old_status = server_status old_task_state = task_state