def _hotplug_server(self): old_floating_ip, server = self.floating_ip_tuple ip_address = old_floating_ip.floating_ip_address private_key = self._get_server_key(server) ssh_client = self.get_remote_client(ip_address, private_key=private_key) old_nic_list = self._get_server_nics(ssh_client) # get a port from a list of one item port_list = self._list_ports(device_id=server['id']) self.assertEqual(1, len(port_list)) old_port = port_list[0] interface = self.interface_client.create_interface( server_id=server['id'], net_id=self.new_net.id)['interfaceAttachment'] self.addCleanup(self.network_client.wait_for_resource_deletion, 'port', interface['port_id'], client=self.ports_client) self.addCleanup(self.delete_wrapper, self.interface_client.delete_interface, server['id'], interface['port_id']) def check_ports(): self.new_port_list = [ port for port in self._list_ports(device_id=server['id']) if port['id'] != old_port['id'] ] return len(self.new_port_list) == 1 if not test.call_until_true(check_ports, CONF.network.build_timeout, CONF.network.build_interval): raise exceptions.TimeoutException( "No new port attached to the server in time (%s sec)! " "Old port: %s. Number of new ports: %d" % (CONF.network.build_timeout, old_port, len( self.new_port_list))) new_port = net_resources.DeletablePort(ports_client=self.ports_client, **self.new_port_list[0]) def check_new_nic(): new_nic_list = self._get_server_nics(ssh_client) self.diff_list = [n for n in new_nic_list if n not in old_nic_list] return len(self.diff_list) == 1 if not test.call_until_true(check_new_nic, CONF.network.build_timeout, CONF.network.build_interval): raise exceptions.TimeoutException("Interface not visible on the " "guest after %s sec" % CONF.network.build_timeout) num, new_nic = self.diff_list[0] ssh_client.assign_static_ip(nic=new_nic, addr=new_port.fixed_ips[0]['ip_address']) ssh_client.set_nic_state(nic=new_nic)
def test_datasource_db_sync_remove(self): # Verify that a replica removes a datasource when a datasource # disappears from the database. CLIENT2_PORT = 4001 client1 = self.admin_manager.congress_client fake_id = self.create_fake(client1) need_to_delete_fake = True try: self.start_replica(CLIENT2_PORT) # Verify that primary server has fake datasource if not test.call_until_true( func=lambda: self.datasource_exists(client1, fake_id), duration=60, sleep_for=1): raise exceptions.TimeoutException( "primary should have fake, but does not") # Create session for second server. client2 = self.create_client(CLIENT2_PORT) # Verify that second server has fake datasource if not test.call_until_true( func=lambda: self.datasource_exists(client2, fake_id), duration=60, sleep_for=1): raise exceptions.TimeoutException( "replica should have fake, but does not") # Remove fake from primary server instance. LOG.debug("removing fake datasource %s", str(fake_id)) client1.delete_datasource(fake_id) need_to_delete_fake = False # Confirm that fake is gone from primary server instance. if not test.call_until_true( func=lambda: self.datasource_missing(client1, fake_id), duration=60, sleep_for=1): self.stop_replica(CLIENT2_PORT) raise exceptions.TimeoutException( "primary instance still has fake") LOG.debug("removed fake datasource from primary instance") # Confirm that second service instance removes fake. if not test.call_until_true( func=lambda: self.datasource_missing(client2, fake_id), duration=60, sleep_for=1): raise exceptions.TimeoutException( "replica should remove fake, but still has it") finally: self.stop_replica(CLIENT2_PORT) if need_to_delete_fake: self.admin_manager.congress_client.delete_datasource(fake_id)
def wait_for_resource_status(self, fetch, status, interval=None, timeout=None): """ @summary: Waits for a network resource to reach a status @param fetch: the callable to be used to query the resource status @type fecth: callable that takes no parameters and returns the resource @param status: the status that the resource has to reach @type status: String @param interval: the number of seconds to wait between each status query @type interval: Integer @param timeout: the maximum number of seconds to wait for the resource to reach the desired status @type timeout: Integer """ if not interval: interval = self.build_interval if not timeout: timeout = self.build_timeout start_time = time.time() while time.time() - start_time <= timeout: resource = fetch() if resource['status'] == status: return time.sleep(interval) # At this point, the wait has timed out message = 'Resource %s' % (str(resource)) message += ' failed to reach status %s' % status message += ' within the required time %s' % timeout caller = misc.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def exec_cmd_on_server_using_fip(self, cmd, ssh_client=None, sub_result=None, expected_value=None): if not ssh_client: ssh_client = self.ssh_client def exec_cmd_and_verify_output(): exec_cmd_retried = 0 while exec_cmd_retried < \ constants.MAX_NO_OF_TIMES_EXECUTION_OVER_SSH: result = ssh_client.exec_command(cmd) self.assertIsNotNone(result) if not result == "": break exec_cmd_retried += 1 time.sleep(constants.INTERVAL_BETWEEN_EXEC_RETRY_ON_SSH) LOG.info("Tried %s times!!!", exec_cmd_retried) if sub_result: msg = ("Failed sub result is not in result Subresult: %r " "Result: %r" % (sub_result, result)) self.assertIn(sub_result, result, msg) if expected_value: msg = ("Failed expected_value is not in result expected_value:" " %r Result: %r" % (expected_value, result)) self.assertEqual(expected_value, result, msg) return result if not test_utils.call_until_true(exec_cmd_and_verify_output, CONF.compute.build_timeout, CONF.compute.build_interval): raise exceptions.TimeoutException("Timed out while waiting to " "execute cmd %s on server. " % cmd)
def test_policy_basic_op(self): self._setup_network_and_servers() body = { "rule": "port_security_group(id, security_group_name) " ":-neutronv2:ports(id, tenant_id, name, network_id," "mac_address, admin_state_up, status, device_id, " "device_owner)," "neutronv2:security_group_port_bindings(id, " "security_group_id), neutronv2:security_groups(" "security_group_id, tenant_id1, security_group_name," "description)" } results = self.admin_manager.congress_client.create_policy_rule( 'classification', body) rule_id = results['id'] self.addCleanup(self.admin_manager.congress_client.delete_policy_rule, 'classification', rule_id) # Find the ports of on this server ports = self._list_ports(device_id=self.servers[0]['id']) def check_data(): results = self.admin_manager.congress_client.list_policy_rows( 'classification', 'port_security_group') for row in results['results']: if (row['data'][0] == ports[0]['id'] and row['data'][1] == self.servers[0]['security_groups'][0]['name']): return True else: return False if not test.call_until_true(func=check_data, duration=20, sleep_for=4): raise exceptions.TimeoutException("Data did not converge in time " "or failure in server")
def wait_for_lease_end(self, lease_id): def check_lease_end(): try: lease = self.reservation_client.get_lease(lease_id)['lease'] if lease: events = lease['events'] return len([ evt for evt in events if evt['event_type'] == 'end_lease' and evt['status'] == 'DONE' ]) > 0 else: LOG.info("Lease with id %s is empty", lease_id) except Exception as e: LOG.info( "Unable to find lease with id %(lease_id)s. " "Exception: %(message)s", { 'lease_id': lease_id, 'message': str(e) }) return True if not test_utils.call_until_true( check_lease_end, CONF.resource_reservation.lease_end_timeout, CONF.resource_reservation.lease_interval): message = "Timed out waiting for lease to change status to DONE" raise exceptions.TimeoutException(message)
def wait_for_resource_status(self, fetch, status): """Waits for a network resource to reach a status @param fetch: the callable to be used to query the resource status @type fecth: callable that takes no parameters and returns the resource @param status: the status that the resource has to reach @type status: String """ interval = self.build_interval timeout = self.build_timeout start_time = time.time() while time.time() - start_time <= timeout: resource = fetch() if resource['status'] == status: return time.sleep(interval) # At this point, the wait has timed out message = 'Resource %s' % (str(resource)) message += ' failed to reach status %s' % status message += ' (current: %s)' % resource['status'] message += ' within the required time %s' % timeout caller = misc.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def test_keystone_roles_table(self): role_schema = ( self.admin_manager.congress_client.show_datasource_table_schema( self.datasource_id, 'roles')['columns']) role_id_col = next(i for i, c in enumerate(role_schema) if c['name'] == 'id') def _check_data_table_keystone_roles(): # Fetch data from keystone each time, because this test may start # before keystone has all the users. roles = self.keystone.list_roles() roles_map = {} for role in roles: roles_map[role['id']] = role results = (self.admin_manager.congress_client.list_datasource_rows( self.datasource_id, 'roles')) for row in results['results']: try: role_row = roles_map[row['data'][role_id_col]] except KeyError: return False for index in range(len(role_schema)): if (str(row['data'][index]) != str( role_row[role_schema[index]['name']])): return False return True if not test.call_until_true(func=_check_data_table_keystone_roles, duration=20, sleep_for=4): raise exceptions.TimeoutException("Data did not converge in time " "or failure in server")
def wait_for_image_status(self, image_id, status): """Waits for a Image to reach a given status.""" start_time = time.time() old_value = value = self._get_image_status(image_id) while True: dtime = time.time() - start_time time.sleep(self.build_interval) if value != old_value: LOG.info( 'Value transition from "%s" to "%s"' 'in %d second(s).', old_value, value, dtime) if value == status: return value if value == 'killed': raise exceptions.ImageKilledException(image_id=image_id, status=status) if dtime > self.build_timeout: message = ('Time Limit Exceeded! (%ds)' 'while waiting for %s, ' 'but we got %s.' % (self.build_timeout, status, value)) caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message) time.sleep(self.build_interval) old_value = value value = self._get_image_status(image_id)
def test_neutronv2_security_group_rules_table(self): security_groups = self.neutron_client.list_security_groups() sgrs_map = {} # security_group_rules for sg in security_groups['security_groups']: for sgr in sg['security_group_rules']: sgrs_map[sgr['id']] = sgr sgrs_schema = ( self.admin_manager.congress_client.show_datasource_table_schema( self.datasource_id, 'security_group_rules')['columns']) def _check_data(): security_group_rules = ( self.admin_manager.congress_client.list_datasource_rows( self.datasource_id, 'security_group_rules')) # Validate security_group_rules table for row in security_group_rules['results']: sg_rule_row = sgrs_map[row['data'][1]] for index in range(len(sgrs_schema)): if (str(row['data'][index]) != str( sg_rule_row[sgrs_schema[index]['name']])): return False return True if not test.call_until_true( func=_check_data, duration=200, sleep_for=10): raise exceptions.TimeoutException("Data did not converge in time " "or failure in server")
def wait_for_image_status(client, image_id, status): """Waits for an image to reach a given status. The client should have a get_image(image_id) method to get the image. The client should also have build_interval and build_timeout attributes. """ resp, image = client.get_image(image_id) start = int(time.time()) while image['status'] != status: time.sleep(client.build_interval) resp, image = client.get_image(image_id) if image['status'] == 'ERROR': raise exceptions.AddImageException(image_id=image_id) # check the status again to avoid a false negative where we hit # the timeout at the same time that the image reached the expected # status if image['status'] == status: return if int(time.time()) - start >= client.build_timeout: message = ('Image %(image_id)s failed to reach %(status)s ' 'status within the required time (%(timeout)s s).' % {'image_id': image_id, 'status': status, 'timeout': client.build_timeout}) message += ' Current status: %s.' % image['status'] raise exceptions.TimeoutException(message)
def _wait_for_resource_status(self, stack_identifier, resource_name, status, failure_pattern='^.*_FAILED$'): """Waits for a Resource to reach a given status.""" fail_regexp = re.compile(failure_pattern) build_timeout = CONF.orchestration.build_timeout build_interval = CONF.orchestration.build_interval start = timeutils.utcnow() while timeutils.delta_seconds(start, timeutils.utcnow()) < build_timeout: try: res = self.client.resources.get( stack_identifier, resource_name) except heat_exceptions.HTTPNotFound: # ignore this, as the resource may not have # been created yet pass else: if res.resource_status == status: return if fail_regexp.search(res.resource_status): raise exceptions.StackResourceBuildErrorException( resource_name=res.resource_name, stack_identifier=stack_identifier, resource_status=res.resource_status, resource_status_reason=res.resource_status_reason) time.sleep(build_interval) message = ('Resource %s failed to reach %s status within ' 'the required time (%s s).' % (res.resource_name, status, build_timeout)) raise exceptions.TimeoutException(message)
def test_glancev2_tags_table(self): def _check_data_table_glance_images(): # Fetch data from glance each time, because this test may start # before glance has all the users. images = self.glancev2.image_list() image_tag_map = {} for image in images: image_tag_map[image['id']] = image['tags'] results = (self.admin_manager.congress_client.list_datasource_rows( 'glancev2', 'tags')) for row in results['results']: image_id, tag = row['data'][0], row['data'][1] glance_image_tags = image_tag_map.get(image_id) if not glance_image_tags: # congress had image that glance doesn't know about. return False if tag not in glance_image_tags: # congress had a tag that wasn't on the image. return False return True if not test.call_until_true(func=_check_data_table_glance_images, duration=20, sleep_for=4): raise exceptions.TimeoutException("Data did not converge in time " "or failure in server")
def wait_for_bm_node_status(client, node_id, attr, status): """Waits for a baremetal node attribute to reach given status. The client should have a show_node(node_uuid) method to get the node. """ _, node = client.show_node(node_id) start = int(time.time()) while node[attr] != status: time.sleep(client.build_interval) _, node = client.show_node(node_id) status_curr = node[attr] if status_curr == status: return if int(time.time()) - start >= client.build_timeout: message = ('Node %(node_id)s failed to reach %(attr)s=%(status)s ' 'within the required time (%(timeout)s s).' % { 'node_id': node_id, 'attr': attr, 'status': status, 'timeout': client.build_timeout }) message += ' Current state of %s: %s.' % (attr, status_curr) caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def wait_for_stack_status(self, stack_identifier, status, failure_status=('CREATE_FAILED', 'DELETE_FAILED', 'UPDATE_FAILED', 'ROLLBACK_FAILED')): """Waits for a Volume to reach a given status.""" stack_status = None start = int(time.time()) while stack_status != status: resp, body = self.get_stack(stack_identifier) stack_name = body['stack_name'] stack_status = body['stack_status'] if stack_status in failure_status: raise exceptions.StackBuildErrorException( stack_identifier=stack_identifier, stack_status=stack_status, stack_status_reason=body['stack_status_reason']) if int(time.time()) - start >= self.build_timeout: message = ('Stack %s failed to reach %s status within ' 'the required time (%s s).' % (stack_name, status, self.build_timeout)) raise exceptions.TimeoutException(message) time.sleep(self.build_interval)
def test_nova_datasource_driver_flavors(self): def _check_data_table_nova_flavors(): # Fetch data from nova each time, because this test may start # before nova has all the users. flavors = self.flavors_client.list_flavors_with_detail() flavor_id_map = {} for flavor in flavors: flavor_id_map[flavor['id']] = flavor results = ( self.admin_manager.congress_client.list_datasource_rows( self.datasource_id, 'flavors')) # TODO(alexsyip): Not sure what the following OS-FLV-EXT-DATA: # prefix is for. keys = ['id', 'name', 'vcpus', 'ram', 'disk', 'OS-FLV-EXT-DATA:ephemeral', 'rxtx_factor'] for row in results['results']: match = True try: flavor_row = flavor_id_map[row['data'][0]] except KeyError: return False for index in range(len(keys)): if row['data'][index] != flavor_row[keys[index]]: match = False break if match: return True return False if not test.call_until_true(func=_check_data_table_nova_flavors, duration=20, sleep_for=4): raise exceptions.TimeoutException("Data did not converge in time " "or failure in server")
def test_resize_server_revert(self): # The server's RAM and disk space should return to its original # values after a resize is reverted previous_flavor_ref, new_flavor_ref = \ self._detect_server_image_flavor(self.server_id) resp, server = self.client.resize(self.server_id, new_flavor_ref) self.assertEqual(202, resp.status) self.client.wait_for_server_status(self.server_id, 'VERIFY_RESIZE') self.client.revert_resize(self.server_id) self.client.wait_for_server_status(self.server_id, 'ACTIVE') # Need to poll for the id change until lp#924371 is fixed resp, server = self.client.get_server(self.server_id) start = int(time.time()) while int(server['flavor']['id']) != previous_flavor_ref: time.sleep(self.build_interval) resp, server = self.client.get_server(self.server_id) if int(time.time()) - start >= self.build_timeout: message = 'Server %s failed to revert resize within the \ required time (%s s).' % (self.server_id, self.build_timeout) raise exceptions.TimeoutException(message)
def wait_for_stack_status(self, stack_identifier, status, failure_pattern='^.*_FAILED$'): """Waits for a Stack to reach a given status.""" start = int(time.time()) fail_regexp = re.compile(failure_pattern) while True: resp, body = self.get_stack(stack_identifier) stack_name = body['stack_name'] stack_status = body['stack_status'] if stack_status == status: return if fail_regexp.search(stack_status): raise exceptions.StackBuildErrorException( stack_identifier=stack_identifier, stack_status=stack_status, stack_status_reason=body['stack_status_reason']) if int(time.time()) - start >= self.build_timeout: message = ('Stack %s failed to reach %s status within ' 'the required time (%s s).' % (stack_name, status, self.build_timeout)) raise exceptions.TimeoutException(message) time.sleep(self.build_interval)
def test_ceilometer_meters_table(self): meter_schema = ( self.admin_manager.congress_client.show_datasource_table_schema( 'ceilometer', 'meters')['columns']) meter_id_col = next(i for i, c in enumerate(meter_schema) if c['name'] == 'meter_id') def _check_data_table_ceilometer_meters(): # Fetch data from ceilometer each time, because this test may start # before ceilometer has all the users. _, meters = self.telemetry_client.list_meters() meter_map = {} for meter in meters: meter_map[meter['meter_id']] = meter results = ( self.admin_manager.congress_client.list_datasource_rows( 'ceilometer', 'meters')) for row in results['results']: try: meter_row = meter_map[row['data'][meter_id_col]] except KeyError: return False for index in range(len(meter_schema)): if (str(row['data'][index]) != str(meter_row[meter_schema[index]['name']])): return False return True if not test.call_until_true(func=_check_data_table_ceilometer_meters, duration=20, sleep_for=4): raise exceptions.TimeoutException("Data did not converge in time " "or failure in server")
def test_neutronv2_networks_table(self): networks = self.neutron_client.list_networks() network_map = {} for network in networks['networks']: network_map[network['id']] = network network_schema = ( self.admin_manager.congress_client.show_datasource_table_schema( self.datasource_id, 'networks')['columns']) def _check_data(): results = (self.admin_manager.congress_client.list_datasource_rows( self.datasource_id, 'networks')) for row in results['results']: network_row = network_map[row['data'][0]] for index in range(len(network_schema)): if (str(row['data'][index]) != str( network_row[network_schema[index]['name']])): return False return True if not test.call_until_true( func=_check_data, duration=200, sleep_for=10): raise exceptions.TimeoutException("Data did not converge in time " "or failure in server")
def _wait_firewall_group_while(self, firewall_group_id, statuses, not_found_ok=False): start = int(time.time()) if not_found_ok: expected_exceptions = (lib_exc.NotFound) else: expected_exceptions = () while True: try: fwg = self.firewall_groups_client.show_firewall_group( firewall_group_id) except expected_exceptions: break status = fwg['firewall_group']['status'] if status not in statuses: break if (int(time.time()) - start >= self.firewall_groups_client.build_timeout): msg = ("Firewall Group %(firewall_group)s failed to reach " "non PENDING status (current %(status)s)") % { "firewall_group": firewall_group_id, "status": status, } raise exceptions.TimeoutException(msg) time.sleep(1)
def wait_for_image_status(client, image_id, status): """Waits for an image to reach a given status. The client should have a get_image(image_id) method to get the image. The client should also have build_interval and build_timeout attributes. """ image = client.get_image(image_id) start = int(time.time()) while image['status'] != status: time.sleep(client.build_interval) image = client.get_image(image_id) status_curr = image['status'] if status_curr == 'ERROR': raise exceptions.AddImageException(image_id=image_id) # check the status again to avoid a false negative where we hit # the timeout at the same time that the image reached the expected # status if status_curr == status: return if int(time.time()) - start >= client.build_timeout: message = ('Image %(image_id)s failed to reach %(status)s state' '(current state %(status_curr)s) ' 'within the required time (%(timeout)s s).' % { 'image_id': image_id, 'status': status, 'status_curr': status_curr, 'timeout': client.build_timeout }) caller = misc_utils.find_test_caller() if caller: message = '(%s) %s' % (caller, message) raise exceptions.TimeoutException(message)
def wait_for_resource_status(self, stack_identifier, resource_name, status, failure_pattern='^.*_FAILED$'): """Waits for a Resource to reach a given status.""" start = int(time.time()) fail_regexp = re.compile(failure_pattern) while True: try: resp, body = self.get_resource(stack_identifier, resource_name) except exceptions.NotFound: # ignore this, as the resource may not have # been created yet pass else: resource_name = body['resource_name'] resource_status = body['resource_status'] if resource_status == status: return if fail_regexp.search(resource_status): raise exceptions.StackBuildErrorException( stack_identifier=stack_identifier, resource_status=resource_status, resource_status_reason=body['resource_status_reason']) if int(time.time()) - start >= self.build_timeout: message = ('Resource %s failed to reach %s status within ' 'the required time (%s s).' % (resource_name, status, self.build_timeout)) raise exceptions.TimeoutException(message) time.sleep(self.build_interval)
def wait_for_stack_status(self, stack_identifier, status, failure_pattern='^.*_FAILED$'): """Waits for a Stack to reach a given status.""" start = int(time.time()) fail_regexp = re.compile(failure_pattern) while True: try: body = self.show_stack(stack_identifier)['stack'] except lib_exc.NotFound: if status == 'DELETE_COMPLETE': return stack_name = body['stack_name'] stack_status = body['stack_status'] if stack_status == status: return body if fail_regexp.search(stack_status): raise exceptions.StackBuildErrorException( stack_identifier=stack_identifier, stack_status=stack_status, stack_status_reason=body['stack_status_reason']) if int(time.time()) - start >= self.build_timeout: message = ( 'Stack %s failed to reach %s status (current: %s) ' 'within the required time (%s s).' % (stack_name, status, stack_status, self.build_timeout)) raise exceptions.TimeoutException(message) time.sleep(self.build_interval)
def wait_for_port_detach(self, port_id): """Waits for the port's device_id to be unset. :param port_id: The id of the port being detached. :returns: The final port dict from the show_port response. """ port = self.ports_client.show_port(port_id)['port'] device_id = port['device_id'] start = int(time.time()) # NOTE(mriedem): Nova updates the port's device_id to '' rather than # None, but it's not contractual so handle Falsey either way. while device_id: time.sleep(self.build_interval) port = self.ports_client.show_port(port_id)['port'] device_id = port['device_id'] timed_out = int(time.time()) - start >= self.build_timeout if device_id and timed_out: message = ('Port %s failed to detach (device_id %s) within ' 'the required time (%s s).' % (port_id, device_id, self.build_timeout)) raise exceptions.TimeoutException(message) return port
def _check_connection(self, vip): timeout = CONF.compute.ping_timeout start = time.time() while not self._connect_to_server(vip.address): # Allow some time for busybox nc to be alive time.sleep(1) if (time.time() - start) > timeout: message = "Timeout out trying to connecto to %s" % vip.address raise exceptions.TimeoutException(message)
def _http_request(self, url, method, **kwargs): """Send an http request with the specified characteristics. Wrapper around httplib.HTTP(S)Connection.request to handle tasks such as setting headers and error handling. """ # Copy the kwargs so we can reuse the original in case of redirects kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {})) kwargs['headers'].setdefault('User-Agent', USER_AGENT) if self.auth_token: kwargs['headers'].setdefault('X-Auth-Token', self.auth_token) self._log_request(method, url, kwargs['headers']) conn = self.get_connection() try: conn_url = posixpath.normpath('%s/%s' % (self.endpoint_path, url)) if kwargs['headers'].get('Transfer-Encoding') == 'chunked': conn.putrequest(method, conn_url) for header, value in kwargs['headers'].items(): conn.putheader(header, value) conn.endheaders() chunk = kwargs['body'].read(CHUNKSIZE) # Chunk it, baby... while chunk: conn.send('%x\r\n%s\r\n' % (len(chunk), chunk)) chunk = kwargs['body'].read(CHUNKSIZE) conn.send('0\r\n\r\n') else: conn.request(method, conn_url, **kwargs) resp = conn.getresponse() except socket.gaierror as e: message = ("Error finding address for %(url)s: %(e)s" % { 'url': url, 'e': e }) raise exc.EndpointNotFound(message) except (socket.error, socket.timeout) as e: message = ("Error communicating with %(endpoint)s %(e)s" % { 'endpoint': self.endpoint, 'e': e }) raise exc.TimeoutException(message) body_iter = ResponseBodyIterator(resp) # Read body into string if it isn't obviously image data if resp.getheader('content-type', None) != 'application/octet-stream': body_str = ''.join([body_chunk for body_chunk in body_iter]) body_iter = StringIO.StringIO(body_str) self._log_response(resp, None) else: self._log_response(resp, body_iter) return resp, body_iter
def _wait_for_active(self, fw_id): def _wait(): _, firewall = self.client.show_firewall(fw_id) firewall = firewall['firewall'] return firewall['status'] == 'ACTIVE' if not test.call_until_true(_wait, CONF.network.build_timeout, CONF.network.build_interval): m = 'Timed out waiting for firewall %s to become ACTIVE.' % fw_id raise exceptions.TimeoutException(m)
def wait_for_server_status(client, server_id, status, ready_wait=True): """Waits for a server to reach a given status.""" def _get_task_state(body): task_state = body.get('OS-EXT-STS:task_state', None) return task_state # NOTE(afazekas): UNKNOWN status possible on ERROR # or in a very early stage. resp, body = client.get_server(server_id) old_status = server_status = body['status'] old_task_state = task_state = _get_task_state(body) start_time = int(time.time()) while True: # NOTE(afazekas): Now the BUILD status only reached # between the UNKOWN->ACTIVE transition. # TODO(afazekas): enumerate and validate the stable status set if status == 'BUILD' and server_status != 'UNKNOWN': return if server_status == status: if ready_wait: if status == 'BUILD': return # NOTE(afazekas): The instance is in "ready for action state" # when no task in progress # NOTE(afazekas): Converted to string bacuse of the XML # responses if str(task_state) == "None": # without state api extension 3 sec usually enough time.sleep(CONFIG.compute.ready_wait) return else: return time.sleep(client.build_interval) resp, body = client.get_server(server_id) server_status = body['status'] task_state = _get_task_state(body) if (server_status != old_status) or (task_state != old_task_state): LOG.info('State transition "%s" ==> "%s" after %d second wait', '/'.join((old_status, str(old_task_state))), '/'.join( (server_status, str(task_state))), time.time() - start_time) if server_status == 'ERROR': raise exceptions.BuildErrorException(server_id=server_id) timed_out = int(time.time()) - start_time >= client.build_timeout if timed_out: message = ('Server %s failed to reach %s status within the ' 'required time (%s s).' % (server_id, status, client.build_timeout)) message += ' Current status: %s.' % server_status raise exceptions.TimeoutException(message) old_status = server_status old_task_state = task_state
def exec_command(self, cmd, cmd_timeout=0): """ Execute the specified command on the server. Note that this method is reading whole command outputs to memory, thus shouldn't be used for large outputs. :returns: data read from standard output of the command. :raises: SSHExecCommandFailed if command returns nonzero status. The exception contains command status stderr content. """ if not self.keep_connection or \ (self.keep_connection and self.ssh_connection is None): self.ssh_connection = self._get_ssh_connection() transport = self.ssh_connection.get_transport() channel = transport.open_session() channel.fileno() # Register event pipe channel.exec_command(cmd) channel.shutdown_write() out_data = [] err_data = [] poll = select.poll() poll.register(channel, select.POLLIN) start_time = time.time() LOG.info("executing cmd: %s" % cmd) while True: if self._is_timed_out(start_time, cmd_timeout): raise exceptions.TimeoutException( "Command: '{0}' executed on host '{1}'.".format( cmd, self.host)) ready = poll.poll(self.channel_timeout) if not any(ready): continue if not ready[0]: # If there is nothing to read. continue out_chunk = err_chunk = None if channel.recv_ready(): out_chunk = channel.recv(self.buf_size) out_data += out_chunk, if channel.recv_stderr_ready(): err_chunk = channel.recv_stderr(self.buf_size) err_data += err_chunk, if channel.closed and not err_chunk and not out_chunk: break # LOG.info("ERROR: {0}".format(err_chunk)) # LOG.info("STUFF: {0}".format(out_chunk)) # hack for avoiding unexpected hung time.sleep(0.1) exit_status = channel.recv_exit_status() if 0 != exit_status: raise SSHExecCommandFailed(command=cmd, exit_status=exit_status, strerror=''.join(err_data)) return ''.join(out_data)