def _hotplug_server(self):
old_floating_ip, server = self.floating_ip_tuple
ip_address = old_floating_ip.floating_ip_address
private_key = self._get_server_key(server)
ssh_client = self.get_remote_client(ip_address,
private_key=private_key)
old_nic_list = self._get_server_nics(ssh_client)
# get a port from a list of one item
port_list = self._list_ports(device_id=server['id'])
self.assertEqual(1, len(port_list))
old_port = port_list[0]
interface = self.interface_client.create_interface(
server_id=server['id'],
net_id=self.new_net.id)['interfaceAttachment']
self.addCleanup(self.network_client.wait_for_resource_deletion,
'port',
interface['port_id'],
client=self.ports_client)
self.addCleanup(self.delete_wrapper,
self.interface_client.delete_interface, server['id'],
interface['port_id'])
def check_ports():
self.new_port_list = [
port for port in self._list_ports(device_id=server['id'])
if port['id'] != old_port['id']
]
return len(self.new_port_list) == 1
if not test.call_until_true(check_ports, CONF.network.build_timeout,
CONF.network.build_interval):
raise exceptions.TimeoutException(
"No new port attached to the server in time (%s sec)! "
"Old port: %s. Number of new ports: %d" %
(CONF.network.build_timeout, old_port, len(
self.new_port_list)))
new_port = net_resources.DeletablePort(ports_client=self.ports_client,
**self.new_port_list[0])
def check_new_nic():
new_nic_list = self._get_server_nics(ssh_client)
self.diff_list = [n for n in new_nic_list if n not in old_nic_list]
return len(self.diff_list) == 1
if not test.call_until_true(check_new_nic, CONF.network.build_timeout,
CONF.network.build_interval):
raise exceptions.TimeoutException("Interface not visible on the "
"guest after %s sec" %
CONF.network.build_timeout)
num, new_nic = self.diff_list[0]
ssh_client.assign_static_ip(nic=new_nic,
addr=new_port.fixed_ips[0]['ip_address'])
ssh_client.set_nic_state(nic=new_nic)
def test_datasource_db_sync_remove(self):
# Verify that a replica removes a datasource when a datasource
# disappears from the database.
CLIENT2_PORT = 4001
client1 = self.admin_manager.congress_client
fake_id = self.create_fake(client1)
need_to_delete_fake = True
try:
self.start_replica(CLIENT2_PORT)
# Verify that primary server has fake datasource
if not test.call_until_true(
func=lambda: self.datasource_exists(client1, fake_id),
duration=60, sleep_for=1):
raise exceptions.TimeoutException(
"primary should have fake, but does not")
# Create session for second server.
client2 = self.create_client(CLIENT2_PORT)
# Verify that second server has fake datasource
if not test.call_until_true(
func=lambda: self.datasource_exists(client2, fake_id),
duration=60, sleep_for=1):
raise exceptions.TimeoutException(
"replica should have fake, but does not")
# Remove fake from primary server instance.
LOG.debug("removing fake datasource %s", str(fake_id))
client1.delete_datasource(fake_id)
need_to_delete_fake = False
# Confirm that fake is gone from primary server instance.
if not test.call_until_true(
func=lambda: self.datasource_missing(client1, fake_id),
duration=60, sleep_for=1):
self.stop_replica(CLIENT2_PORT)
raise exceptions.TimeoutException(
"primary instance still has fake")
LOG.debug("removed fake datasource from primary instance")
# Confirm that second service instance removes fake.
if not test.call_until_true(
func=lambda: self.datasource_missing(client2, fake_id),
duration=60, sleep_for=1):
raise exceptions.TimeoutException(
"replica should remove fake, but still has it")
finally:
self.stop_replica(CLIENT2_PORT)
if need_to_delete_fake:
self.admin_manager.congress_client.delete_datasource(fake_id)
def wait_for_resource_status(self, fetch, status, interval=None,
timeout=None):
"""
@summary: Waits for a network resource to reach a status
@param fetch: the callable to be used to query the resource status
@type fecth: callable that takes no parameters and returns the resource
@param status: the status that the resource has to reach
@type status: String
@param interval: the number of seconds to wait between each status
query
@type interval: Integer
@param timeout: the maximum number of seconds to wait for the resource
to reach the desired status
@type timeout: Integer
"""
if not interval:
interval = self.build_interval
if not timeout:
timeout = self.build_timeout
start_time = time.time()
while time.time() - start_time <= timeout:
resource = fetch()
if resource['status'] == status:
return
time.sleep(interval)
# At this point, the wait has timed out
message = 'Resource %s' % (str(resource))
message += ' failed to reach status %s' % status
message += ' within the required time %s' % timeout
caller = misc.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
def exec_cmd_on_server_using_fip(self,
cmd,
ssh_client=None,
sub_result=None,
expected_value=None):
if not ssh_client:
ssh_client = self.ssh_client
def exec_cmd_and_verify_output():
exec_cmd_retried = 0
while exec_cmd_retried < \
constants.MAX_NO_OF_TIMES_EXECUTION_OVER_SSH:
result = ssh_client.exec_command(cmd)
self.assertIsNotNone(result)
if not result == "":
break
exec_cmd_retried += 1
time.sleep(constants.INTERVAL_BETWEEN_EXEC_RETRY_ON_SSH)
LOG.info("Tried %s times!!!", exec_cmd_retried)
if sub_result:
msg = ("Failed sub result is not in result Subresult: %r "
"Result: %r" % (sub_result, result))
self.assertIn(sub_result, result, msg)
if expected_value:
msg = ("Failed expected_value is not in result expected_value:"
" %r Result: %r" % (expected_value, result))
self.assertEqual(expected_value, result, msg)
return result
if not test_utils.call_until_true(exec_cmd_and_verify_output,
CONF.compute.build_timeout,
CONF.compute.build_interval):
raise exceptions.TimeoutException("Timed out while waiting to "
"execute cmd %s on server. " %
cmd)
def test_policy_basic_op(self):
self._setup_network_and_servers()
body = {
"rule":
"port_security_group(id, security_group_name) "
":-neutronv2:ports(id, tenant_id, name, network_id,"
"mac_address, admin_state_up, status, device_id, "
"device_owner),"
"neutronv2:security_group_port_bindings(id, "
"security_group_id), neutronv2:security_groups("
"security_group_id, tenant_id1, security_group_name,"
"description)"
}
results = self.admin_manager.congress_client.create_policy_rule(
'classification', body)
rule_id = results['id']
self.addCleanup(self.admin_manager.congress_client.delete_policy_rule,
'classification', rule_id)
# Find the ports of on this server
ports = self._list_ports(device_id=self.servers[0]['id'])
def check_data():
results = self.admin_manager.congress_client.list_policy_rows(
'classification', 'port_security_group')
for row in results['results']:
if (row['data'][0] == ports[0]['id'] and row['data'][1]
== self.servers[0]['security_groups'][0]['name']):
return True
else:
return False
if not test.call_until_true(func=check_data, duration=20, sleep_for=4):
raise exceptions.TimeoutException("Data did not converge in time "
"or failure in server")
def wait_for_lease_end(self, lease_id):
def check_lease_end():
try:
lease = self.reservation_client.get_lease(lease_id)['lease']
if lease:
events = lease['events']
return len([
evt
for evt in events if evt['event_type'] == 'end_lease'
and evt['status'] == 'DONE'
]) > 0
else:
LOG.info("Lease with id %s is empty", lease_id)
except Exception as e:
LOG.info(
"Unable to find lease with id %(lease_id)s. "
"Exception: %(message)s", {
'lease_id': lease_id,
'message': str(e)
})
return True
if not test_utils.call_until_true(
check_lease_end, CONF.resource_reservation.lease_end_timeout,
CONF.resource_reservation.lease_interval):
message = "Timed out waiting for lease to change status to DONE"
raise exceptions.TimeoutException(message)
def wait_for_resource_status(self, fetch, status):
"""Waits for a network resource to reach a status
@param fetch: the callable to be used to query the resource status
@type fecth: callable that takes no parameters and returns the resource
@param status: the status that the resource has to reach
@type status: String
"""
interval = self.build_interval
timeout = self.build_timeout
start_time = time.time()
while time.time() - start_time <= timeout:
resource = fetch()
if resource['status'] == status:
return
time.sleep(interval)
# At this point, the wait has timed out
message = 'Resource %s' % (str(resource))
message += ' failed to reach status %s' % status
message += ' (current: %s)' % resource['status']
message += ' within the required time %s' % timeout
caller = misc.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
def test_keystone_roles_table(self):
role_schema = (
self.admin_manager.congress_client.show_datasource_table_schema(
self.datasource_id, 'roles')['columns'])
role_id_col = next(i for i, c in enumerate(role_schema)
if c['name'] == 'id')
def _check_data_table_keystone_roles():
# Fetch data from keystone each time, because this test may start
# before keystone has all the users.
roles = self.keystone.list_roles()
roles_map = {}
for role in roles:
roles_map[role['id']] = role
results = (self.admin_manager.congress_client.list_datasource_rows(
self.datasource_id, 'roles'))
for row in results['results']:
try:
role_row = roles_map[row['data'][role_id_col]]
except KeyError:
return False
for index in range(len(role_schema)):
if (str(row['data'][index]) != str(
role_row[role_schema[index]['name']])):
return False
return True
if not test.call_until_true(func=_check_data_table_keystone_roles,
duration=20,
sleep_for=4):
raise exceptions.TimeoutException("Data did not converge in time "
"or failure in server")
def wait_for_image_status(self, image_id, status):
"""Waits for a Image to reach a given status."""
start_time = time.time()
old_value = value = self._get_image_status(image_id)
while True:
dtime = time.time() - start_time
time.sleep(self.build_interval)
if value != old_value:
LOG.info(
'Value transition from "%s" to "%s"'
'in %d second(s).', old_value, value, dtime)
if value == status:
return value
if value == 'killed':
raise exceptions.ImageKilledException(image_id=image_id,
status=status)
if dtime > self.build_timeout:
message = ('Time Limit Exceeded! (%ds)'
'while waiting for %s, '
'but we got %s.' %
(self.build_timeout, status, value))
caller = misc_utils.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
time.sleep(self.build_interval)
old_value = value
value = self._get_image_status(image_id)
def test_neutronv2_security_group_rules_table(self):
security_groups = self.neutron_client.list_security_groups()
sgrs_map = {} # security_group_rules
for sg in security_groups['security_groups']:
for sgr in sg['security_group_rules']:
sgrs_map[sgr['id']] = sgr
sgrs_schema = (
self.admin_manager.congress_client.show_datasource_table_schema(
self.datasource_id, 'security_group_rules')['columns'])
def _check_data():
security_group_rules = (
self.admin_manager.congress_client.list_datasource_rows(
self.datasource_id, 'security_group_rules'))
# Validate security_group_rules table
for row in security_group_rules['results']:
sg_rule_row = sgrs_map[row['data'][1]]
for index in range(len(sgrs_schema)):
if (str(row['data'][index]) != str(
sg_rule_row[sgrs_schema[index]['name']])):
return False
return True
if not test.call_until_true(
func=_check_data, duration=200, sleep_for=10):
raise exceptions.TimeoutException("Data did not converge in time "
"or failure in server")
def wait_for_image_status(client, image_id, status):
"""Waits for an image to reach a given status.
The client should have a get_image(image_id) method to get the image.
The client should also have build_interval and build_timeout attributes.
"""
resp, image = client.get_image(image_id)
start = int(time.time())
while image['status'] != status:
time.sleep(client.build_interval)
resp, image = client.get_image(image_id)
if image['status'] == 'ERROR':
raise exceptions.AddImageException(image_id=image_id)
# check the status again to avoid a false negative where we hit
# the timeout at the same time that the image reached the expected
# status
if image['status'] == status:
return
if int(time.time()) - start >= client.build_timeout:
message = ('Image %(image_id)s failed to reach %(status)s '
'status within the required time (%(timeout)s s).' %
{'image_id': image_id,
'status': status,
'timeout': client.build_timeout})
message += ' Current status: %s.' % image['status']
raise exceptions.TimeoutException(message)
def _wait_for_resource_status(self, stack_identifier, resource_name,
status, failure_pattern='^.*_FAILED$'):
"""Waits for a Resource to reach a given status."""
fail_regexp = re.compile(failure_pattern)
build_timeout = CONF.orchestration.build_timeout
build_interval = CONF.orchestration.build_interval
start = timeutils.utcnow()
while timeutils.delta_seconds(start,
timeutils.utcnow()) < build_timeout:
try:
res = self.client.resources.get(
stack_identifier, resource_name)
except heat_exceptions.HTTPNotFound:
# ignore this, as the resource may not have
# been created yet
pass
else:
if res.resource_status == status:
return
if fail_regexp.search(res.resource_status):
raise exceptions.StackResourceBuildErrorException(
resource_name=res.resource_name,
stack_identifier=stack_identifier,
resource_status=res.resource_status,
resource_status_reason=res.resource_status_reason)
time.sleep(build_interval)
message = ('Resource %s failed to reach %s status within '
'the required time (%s s).' %
(res.resource_name, status, build_timeout))
raise exceptions.TimeoutException(message)
def test_glancev2_tags_table(self):
def _check_data_table_glance_images():
# Fetch data from glance each time, because this test may start
# before glance has all the users.
images = self.glancev2.image_list()
image_tag_map = {}
for image in images:
image_tag_map[image['id']] = image['tags']
results = (self.admin_manager.congress_client.list_datasource_rows(
'glancev2', 'tags'))
for row in results['results']:
image_id, tag = row['data'][0], row['data'][1]
glance_image_tags = image_tag_map.get(image_id)
if not glance_image_tags:
# congress had image that glance doesn't know about.
return False
if tag not in glance_image_tags:
# congress had a tag that wasn't on the image.
return False
return True
if not test.call_until_true(func=_check_data_table_glance_images,
duration=20,
sleep_for=4):
raise exceptions.TimeoutException("Data did not converge in time "
"or failure in server")
def wait_for_bm_node_status(client, node_id, attr, status):
"""Waits for a baremetal node attribute to reach given status.
The client should have a show_node(node_uuid) method to get the node.
"""
_, node = client.show_node(node_id)
start = int(time.time())
while node[attr] != status:
time.sleep(client.build_interval)
_, node = client.show_node(node_id)
status_curr = node[attr]
if status_curr == status:
return
if int(time.time()) - start >= client.build_timeout:
message = ('Node %(node_id)s failed to reach %(attr)s=%(status)s '
'within the required time (%(timeout)s s).' % {
'node_id': node_id,
'attr': attr,
'status': status,
'timeout': client.build_timeout
})
message += ' Current state of %s: %s.' % (attr, status_curr)
caller = misc_utils.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
def wait_for_stack_status(self,
stack_identifier,
status,
failure_status=('CREATE_FAILED', 'DELETE_FAILED',
'UPDATE_FAILED',
'ROLLBACK_FAILED')):
"""Waits for a Volume to reach a given status."""
stack_status = None
start = int(time.time())
while stack_status != status:
resp, body = self.get_stack(stack_identifier)
stack_name = body['stack_name']
stack_status = body['stack_status']
if stack_status in failure_status:
raise exceptions.StackBuildErrorException(
stack_identifier=stack_identifier,
stack_status=stack_status,
stack_status_reason=body['stack_status_reason'])
if int(time.time()) - start >= self.build_timeout:
message = ('Stack %s failed to reach %s status within '
'the required time (%s s).' %
(stack_name, status, self.build_timeout))
raise exceptions.TimeoutException(message)
time.sleep(self.build_interval)
def test_nova_datasource_driver_flavors(self):
def _check_data_table_nova_flavors():
# Fetch data from nova each time, because this test may start
# before nova has all the users.
flavors = self.flavors_client.list_flavors_with_detail()
flavor_id_map = {}
for flavor in flavors:
flavor_id_map[flavor['id']] = flavor
results = (
self.admin_manager.congress_client.list_datasource_rows(
self.datasource_id, 'flavors'))
# TODO(alexsyip): Not sure what the following OS-FLV-EXT-DATA:
# prefix is for.
keys = ['id', 'name', 'vcpus', 'ram', 'disk',
'OS-FLV-EXT-DATA:ephemeral', 'rxtx_factor']
for row in results['results']:
match = True
try:
flavor_row = flavor_id_map[row['data'][0]]
except KeyError:
return False
for index in range(len(keys)):
if row['data'][index] != flavor_row[keys[index]]:
match = False
break
if match:
return True
return False
if not test.call_until_true(func=_check_data_table_nova_flavors,
duration=20, sleep_for=4):
raise exceptions.TimeoutException("Data did not converge in time "
"or failure in server")
def test_resize_server_revert(self):
# The server's RAM and disk space should return to its original
# values after a resize is reverted
previous_flavor_ref, new_flavor_ref = \
self._detect_server_image_flavor(self.server_id)
resp, server = self.client.resize(self.server_id, new_flavor_ref)
self.assertEqual(202, resp.status)
self.client.wait_for_server_status(self.server_id, 'VERIFY_RESIZE')
self.client.revert_resize(self.server_id)
self.client.wait_for_server_status(self.server_id, 'ACTIVE')
# Need to poll for the id change until lp#924371 is fixed
resp, server = self.client.get_server(self.server_id)
start = int(time.time())
while int(server['flavor']['id']) != previous_flavor_ref:
time.sleep(self.build_interval)
resp, server = self.client.get_server(self.server_id)
if int(time.time()) - start >= self.build_timeout:
message = 'Server %s failed to revert resize within the \
required time (%s s).' % (self.server_id, self.build_timeout)
raise exceptions.TimeoutException(message)
def wait_for_stack_status(self,
stack_identifier,
status,
failure_pattern='^.*_FAILED$'):
"""Waits for a Stack to reach a given status."""
start = int(time.time())
fail_regexp = re.compile(failure_pattern)
while True:
resp, body = self.get_stack(stack_identifier)
stack_name = body['stack_name']
stack_status = body['stack_status']
if stack_status == status:
return
if fail_regexp.search(stack_status):
raise exceptions.StackBuildErrorException(
stack_identifier=stack_identifier,
stack_status=stack_status,
stack_status_reason=body['stack_status_reason'])
if int(time.time()) - start >= self.build_timeout:
message = ('Stack %s failed to reach %s status within '
'the required time (%s s).' %
(stack_name, status, self.build_timeout))
raise exceptions.TimeoutException(message)
time.sleep(self.build_interval)
def test_ceilometer_meters_table(self):
meter_schema = (
self.admin_manager.congress_client.show_datasource_table_schema(
'ceilometer', 'meters')['columns'])
meter_id_col = next(i for i, c in enumerate(meter_schema)
if c['name'] == 'meter_id')
def _check_data_table_ceilometer_meters():
# Fetch data from ceilometer each time, because this test may start
# before ceilometer has all the users.
_, meters = self.telemetry_client.list_meters()
meter_map = {}
for meter in meters:
meter_map[meter['meter_id']] = meter
results = (
self.admin_manager.congress_client.list_datasource_rows(
'ceilometer', 'meters'))
for row in results['results']:
try:
meter_row = meter_map[row['data'][meter_id_col]]
except KeyError:
return False
for index in range(len(meter_schema)):
if (str(row['data'][index]) !=
str(meter_row[meter_schema[index]['name']])):
return False
return True
if not test.call_until_true(func=_check_data_table_ceilometer_meters,
duration=20, sleep_for=4):
raise exceptions.TimeoutException("Data did not converge in time "
"or failure in server")
def test_neutronv2_networks_table(self):
networks = self.neutron_client.list_networks()
network_map = {}
for network in networks['networks']:
network_map[network['id']] = network
network_schema = (
self.admin_manager.congress_client.show_datasource_table_schema(
self.datasource_id, 'networks')['columns'])
def _check_data():
results = (self.admin_manager.congress_client.list_datasource_rows(
self.datasource_id, 'networks'))
for row in results['results']:
network_row = network_map[row['data'][0]]
for index in range(len(network_schema)):
if (str(row['data'][index]) != str(
network_row[network_schema[index]['name']])):
return False
return True
if not test.call_until_true(
func=_check_data, duration=200, sleep_for=10):
raise exceptions.TimeoutException("Data did not converge in time "
"or failure in server")
def _wait_firewall_group_while(self,
firewall_group_id,
statuses,
not_found_ok=False):
start = int(time.time())
if not_found_ok:
expected_exceptions = (lib_exc.NotFound)
else:
expected_exceptions = ()
while True:
try:
fwg = self.firewall_groups_client.show_firewall_group(
firewall_group_id)
except expected_exceptions:
break
status = fwg['firewall_group']['status']
if status not in statuses:
break
if (int(time.time()) - start >=
self.firewall_groups_client.build_timeout):
msg = ("Firewall Group %(firewall_group)s failed to reach "
"non PENDING status (current %(status)s)") % {
"firewall_group": firewall_group_id,
"status": status,
}
raise exceptions.TimeoutException(msg)
time.sleep(1)
def wait_for_image_status(client, image_id, status):
"""Waits for an image to reach a given status.
The client should have a get_image(image_id) method to get the image.
The client should also have build_interval and build_timeout attributes.
"""
image = client.get_image(image_id)
start = int(time.time())
while image['status'] != status:
time.sleep(client.build_interval)
image = client.get_image(image_id)
status_curr = image['status']
if status_curr == 'ERROR':
raise exceptions.AddImageException(image_id=image_id)
# check the status again to avoid a false negative where we hit
# the timeout at the same time that the image reached the expected
# status
if status_curr == status:
return
if int(time.time()) - start >= client.build_timeout:
message = ('Image %(image_id)s failed to reach %(status)s state'
'(current state %(status_curr)s) '
'within the required time (%(timeout)s s).' % {
'image_id': image_id,
'status': status,
'status_curr': status_curr,
'timeout': client.build_timeout
})
caller = misc_utils.find_test_caller()
if caller:
message = '(%s) %s' % (caller, message)
raise exceptions.TimeoutException(message)
def wait_for_resource_status(self,
stack_identifier,
resource_name,
status,
failure_pattern='^.*_FAILED$'):
"""Waits for a Resource to reach a given status."""
start = int(time.time())
fail_regexp = re.compile(failure_pattern)
while True:
try:
resp, body = self.get_resource(stack_identifier, resource_name)
except exceptions.NotFound:
# ignore this, as the resource may not have
# been created yet
pass
else:
resource_name = body['resource_name']
resource_status = body['resource_status']
if resource_status == status:
return
if fail_regexp.search(resource_status):
raise exceptions.StackBuildErrorException(
stack_identifier=stack_identifier,
resource_status=resource_status,
resource_status_reason=body['resource_status_reason'])
if int(time.time()) - start >= self.build_timeout:
message = ('Resource %s failed to reach %s status within '
'the required time (%s s).' %
(resource_name, status, self.build_timeout))
raise exceptions.TimeoutException(message)
time.sleep(self.build_interval)
def wait_for_stack_status(self,
stack_identifier,
status,
failure_pattern='^.*_FAILED$'):
"""Waits for a Stack to reach a given status."""
start = int(time.time())
fail_regexp = re.compile(failure_pattern)
while True:
try:
body = self.show_stack(stack_identifier)['stack']
except lib_exc.NotFound:
if status == 'DELETE_COMPLETE':
return
stack_name = body['stack_name']
stack_status = body['stack_status']
if stack_status == status:
return body
if fail_regexp.search(stack_status):
raise exceptions.StackBuildErrorException(
stack_identifier=stack_identifier,
stack_status=stack_status,
stack_status_reason=body['stack_status_reason'])
if int(time.time()) - start >= self.build_timeout:
message = (
'Stack %s failed to reach %s status (current: %s) '
'within the required time (%s s).' %
(stack_name, status, stack_status, self.build_timeout))
raise exceptions.TimeoutException(message)
time.sleep(self.build_interval)
def wait_for_port_detach(self, port_id):
"""Waits for the port's device_id to be unset.
:param port_id: The id of the port being detached.
:returns: The final port dict from the show_port response.
"""
port = self.ports_client.show_port(port_id)['port']
device_id = port['device_id']
start = int(time.time())
# NOTE(mriedem): Nova updates the port's device_id to '' rather than
# None, but it's not contractual so handle Falsey either way.
while device_id:
time.sleep(self.build_interval)
port = self.ports_client.show_port(port_id)['port']
device_id = port['device_id']
timed_out = int(time.time()) - start >= self.build_timeout
if device_id and timed_out:
message = ('Port %s failed to detach (device_id %s) within '
'the required time (%s s).' %
(port_id, device_id, self.build_timeout))
raise exceptions.TimeoutException(message)
return port
def _check_connection(self, vip):
timeout = CONF.compute.ping_timeout
start = time.time()
while not self._connect_to_server(vip.address):
# Allow some time for busybox nc to be alive
time.sleep(1)
if (time.time() - start) > timeout:
message = "Timeout out trying to connecto to %s" % vip.address
raise exceptions.TimeoutException(message)
def _http_request(self, url, method, **kwargs):
"""Send an http request with the specified characteristics.
Wrapper around httplib.HTTP(S)Connection.request to handle tasks such
as setting headers and error handling.
"""
# Copy the kwargs so we can reuse the original in case of redirects
kwargs['headers'] = copy.deepcopy(kwargs.get('headers', {}))
kwargs['headers'].setdefault('User-Agent', USER_AGENT)
if self.auth_token:
kwargs['headers'].setdefault('X-Auth-Token', self.auth_token)
self._log_request(method, url, kwargs['headers'])
conn = self.get_connection()
try:
conn_url = posixpath.normpath('%s/%s' % (self.endpoint_path, url))
if kwargs['headers'].get('Transfer-Encoding') == 'chunked':
conn.putrequest(method, conn_url)
for header, value in kwargs['headers'].items():
conn.putheader(header, value)
conn.endheaders()
chunk = kwargs['body'].read(CHUNKSIZE)
# Chunk it, baby...
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = kwargs['body'].read(CHUNKSIZE)
conn.send('0\r\n\r\n')
else:
conn.request(method, conn_url, **kwargs)
resp = conn.getresponse()
except socket.gaierror as e:
message = ("Error finding address for %(url)s: %(e)s" % {
'url': url,
'e': e
})
raise exc.EndpointNotFound(message)
except (socket.error, socket.timeout) as e:
message = ("Error communicating with %(endpoint)s %(e)s" % {
'endpoint': self.endpoint,
'e': e
})
raise exc.TimeoutException(message)
body_iter = ResponseBodyIterator(resp)
# Read body into string if it isn't obviously image data
if resp.getheader('content-type', None) != 'application/octet-stream':
body_str = ''.join([body_chunk for body_chunk in body_iter])
body_iter = StringIO.StringIO(body_str)
self._log_response(resp, None)
else:
self._log_response(resp, body_iter)
return resp, body_iter
def _wait_for_active(self, fw_id):
def _wait():
_, firewall = self.client.show_firewall(fw_id)
firewall = firewall['firewall']
return firewall['status'] == 'ACTIVE'
if not test.call_until_true(_wait, CONF.network.build_timeout,
CONF.network.build_interval):
m = 'Timed out waiting for firewall %s to become ACTIVE.' % fw_id
raise exceptions.TimeoutException(m)
def wait_for_server_status(client, server_id, status, ready_wait=True):
"""Waits for a server to reach a given status."""
def _get_task_state(body):
task_state = body.get('OS-EXT-STS:task_state', None)
return task_state
# NOTE(afazekas): UNKNOWN status possible on ERROR
# or in a very early stage.
resp, body = client.get_server(server_id)
old_status = server_status = body['status']
old_task_state = task_state = _get_task_state(body)
start_time = int(time.time())
while True:
# NOTE(afazekas): Now the BUILD status only reached
# between the UNKOWN->ACTIVE transition.
# TODO(afazekas): enumerate and validate the stable status set
if status == 'BUILD' and server_status != 'UNKNOWN':
return
if server_status == status:
if ready_wait:
if status == 'BUILD':
return
# NOTE(afazekas): The instance is in "ready for action state"
# when no task in progress
# NOTE(afazekas): Converted to string bacuse of the XML
# responses
if str(task_state) == "None":
# without state api extension 3 sec usually enough
time.sleep(CONFIG.compute.ready_wait)
return
else:
return
time.sleep(client.build_interval)
resp, body = client.get_server(server_id)
server_status = body['status']
task_state = _get_task_state(body)
if (server_status != old_status) or (task_state != old_task_state):
LOG.info('State transition "%s" ==> "%s" after %d second wait',
'/'.join((old_status, str(old_task_state))), '/'.join(
(server_status, str(task_state))),
time.time() - start_time)
if server_status == 'ERROR':
raise exceptions.BuildErrorException(server_id=server_id)
timed_out = int(time.time()) - start_time >= client.build_timeout
if timed_out:
message = ('Server %s failed to reach %s status within the '
'required time (%s s).' %
(server_id, status, client.build_timeout))
message += ' Current status: %s.' % server_status
raise exceptions.TimeoutException(message)
old_status = server_status
old_task_state = task_state
def exec_command(self, cmd, cmd_timeout=0):
"""
Execute the specified command on the server.
Note that this method is reading whole command outputs to memory, thus
shouldn't be used for large outputs.
:returns: data read from standard output of the command.
:raises: SSHExecCommandFailed if command returns nonzero
status. The exception contains command status stderr content.
"""
if not self.keep_connection or \
(self.keep_connection and self.ssh_connection is None):
self.ssh_connection = self._get_ssh_connection()
transport = self.ssh_connection.get_transport()
channel = transport.open_session()
channel.fileno() # Register event pipe
channel.exec_command(cmd)
channel.shutdown_write()
out_data = []
err_data = []
poll = select.poll()
poll.register(channel, select.POLLIN)
start_time = time.time()
LOG.info("executing cmd: %s" % cmd)
while True:
if self._is_timed_out(start_time, cmd_timeout):
raise exceptions.TimeoutException(
"Command: '{0}' executed on host '{1}'.".format(
cmd, self.host))
ready = poll.poll(self.channel_timeout)
if not any(ready):
continue
if not ready[0]: # If there is nothing to read.
continue
out_chunk = err_chunk = None
if channel.recv_ready():
out_chunk = channel.recv(self.buf_size)
out_data += out_chunk,
if channel.recv_stderr_ready():
err_chunk = channel.recv_stderr(self.buf_size)
err_data += err_chunk,
if channel.closed and not err_chunk and not out_chunk:
break
# LOG.info("ERROR: {0}".format(err_chunk))
# LOG.info("STUFF: {0}".format(out_chunk))
# hack for avoiding unexpected hung
time.sleep(0.1)
exit_status = channel.recv_exit_status()
if 0 != exit_status:
raise SSHExecCommandFailed(command=cmd,
exit_status=exit_status,
strerror=''.join(err_data))
return ''.join(out_data)
