def test_aggregate_multitenancy_isolation_filter(self):
"""Tests common scenarios with the AggregateMultiTenancyIsolation
filter:
* hosts in a tenant-isolated aggregate are only accepted for that
tenant
* hosts not in a tenant-isolated aggregate are acceptable for all
tenants, including tenants with access to the isolated-tenant
aggregate
"""
# Create a tenant-isolated aggregate for the non-admin user.
user_api = self.useFixture(
nova_fixtures.OSAPIFixture(api_version='v2.1',
project_id=uuids.non_admin)).api
agg_id = self.admin_api.post_aggregate(
{'aggregate': {'name': 'non_admin_agg'}})['id']
meta_req = {'set_metadata': {
'metadata': {'filter_tenant_id': uuids.non_admin}}}
self.admin_api.api_post('/os-aggregates/%s/action' % agg_id, meta_req)
# Add host2 to the aggregate; we'll restrict host2 to the non-admin
# tenant.
host_req = {'add_host': {'host': 'host2'}}
self.admin_api.api_post('/os-aggregates/%s/action' % agg_id, host_req)
# Stub out select_destinations to assert how many host candidates were
# available per tenant-specific request.
original_filtered_hosts = (
nova.scheduler.host_manager.HostManager.get_filtered_hosts)
def spy_get_filtered_hosts(*args, **kwargs):
self.filtered_hosts = original_filtered_hosts(*args, **kwargs)
return self.filtered_hosts
self.stub_out(
'nova.scheduler.host_manager.HostManager.get_filtered_hosts',
spy_get_filtered_hosts)
# Create a server for the admin - should only have one host candidate.
server_req = self._build_minimal_create_server_request(
self.admin_api,
'test_aggregate_multitenancy_isolation_filter-admin',
networks='none') # requires microversion 2.37
server_req = {'server': server_req}
with utils.temporary_mutation(self.admin_api, microversion='2.37'):
server = self.admin_api.post_server(server_req)
server = self._wait_for_state_change(self.admin_api, server, 'ACTIVE')
# Assert it's not on host2 which is isolated to the non-admin tenant.
self.assertNotEqual('host2', server['OS-EXT-SRV-ATTR:host'])
self.assertEqual(1, len(self.filtered_hosts))
# Now create a server for the non-admin tenant to which host2 is
# isolated via the aggregate, but the other compute host is a
# candidate. We don't assert that the non-admin tenant server shows
# up on host2 because the other host, which is not isolated to the
# aggregate, is still a candidate.
server_req = self._build_minimal_create_server_request(
user_api,
'test_aggregate_multitenancy_isolation_filter-user',
networks='none') # requires microversion 2.37
server_req = {'server': server_req}
with utils.temporary_mutation(user_api, microversion='2.37'):
server = user_api.post_server(server_req)
self._wait_for_state_change(user_api, server, 'ACTIVE')
self.assertEqual(2, len(self.filtered_hosts))
def test_aggregate_multitenancy_isolation_filter(self):
"""Tests common scenarios with the AggregateMultiTenancyIsolation
filter:
* hosts in a tenant-isolated aggregate are only accepted for that
tenant
* hosts not in a tenant-isolated aggregate are acceptable for all
tenants, including tenants with access to the isolated-tenant
aggregate
"""
# Create a tenant-isolated aggregate for the non-admin user.
user_api = self.useFixture(
nova_fixtures.OSAPIFixture(api_version='v2.1',
project_id=uuids.non_admin)).api
agg_id = self.admin_api.post_aggregate(
{'aggregate': {'name': 'non_admin_agg'}})['id']
meta_req = {'set_metadata': {
'metadata': {'filter_tenant_id': uuids.non_admin}}}
self.admin_api.api_post('/os-aggregates/%s/action' % agg_id, meta_req)
# Add host2 to the aggregate; we'll restrict host2 to the non-admin
# tenant.
host_req = {'add_host': {'host': 'host2'}}
self.admin_api.api_post('/os-aggregates/%s/action' % agg_id, host_req)
# Stub out select_destinations to assert how many host candidates were
# available per tenant-specific request.
original_filtered_hosts = (
nova.scheduler.host_manager.HostManager.get_filtered_hosts)
def spy_get_filtered_hosts(*args, **kwargs):
self.filtered_hosts = original_filtered_hosts(*args, **kwargs)
return self.filtered_hosts
self.stub_out(
'nova.scheduler.host_manager.HostManager.get_filtered_hosts',
spy_get_filtered_hosts)
# Create a server for the admin - should only have one host candidate.
server_req = self._build_minimal_create_server_request(
self.admin_api,
'test_aggregate_multitenancy_isolation_filter-admin',
networks='none') # requires microversion 2.37
server_req = {'server': server_req}
with utils.temporary_mutation(self.admin_api, microversion='2.37'):
server = self.admin_api.post_server(server_req)
server = self._wait_for_state_change(self.admin_api, server, 'ACTIVE')
# Assert it's not on host2 which is isolated to the non-admin tenant.
self.assertNotEqual('host2', server['OS-EXT-SRV-ATTR:host'])
self.assertEqual(1, len(self.filtered_hosts))
# Now create a server for the non-admin tenant to which host2 is
# isolated via the aggregate, but the other compute host is a
# candidate. We don't assert that the non-admin tenant server shows
# up on host2 because the other host, which is not isolated to the
# aggregate, is still a candidate.
server_req = self._build_minimal_create_server_request(
user_api,
'test_aggregate_multitenancy_isolation_filter-user',
networks='none') # requires microversion 2.37
server_req = {'server': server_req}
with utils.temporary_mutation(user_api, microversion='2.37'):
server = user_api.post_server(server_req)
self._wait_for_state_change(user_api, server, 'ACTIVE')
self.assertEqual(2, len(self.filtered_hosts))
def quota_rollback(self, context, reservations, project_id=None):
# FIXME(comstud): bug 1153795: Conductor manager should accept
# a project_id kwarg to be able to pass to the quota rollback call.
if project_id is None:
project_id = context.project_id
with utils.temporary_mutation(context, project_id=project_id):
return self.conductor_rpcapi.quota_rollback(context, reservations)
def test_bury_in_cell0_instance_create_action(self):
"""Tests creating a server which will fail scheduling because there is
no compute service and result in the instance being created (buried)
in cell0.
"""
server = self._build_server(networks='none')
# Use microversion 2.37 to create a server without any networking.
with utils.temporary_mutation(self.api, microversion='2.37'):
server = self.api.post_server({'server': server})
# The server should go to ERROR status and have a NoValidHost fault.
server = self._wait_for_state_change(server, 'ERROR')
self.assertIn('fault', server)
self.assertIn('No valid host', server['fault']['message'])
self.assertEqual('', server['hostId'])
# Assert the "create" instance action exists and is failed.
actions = self.api.get_instance_actions(server['id'])
self.assertEqual(1, len(actions), actions)
action = actions[0]
self.assertEqual(instance_actions.CREATE, action['action'])
self.assertEqual('Error', action['message'])
# Get the events. There should be one with an Error result.
action = self.api.api_get(
'/servers/%s/os-instance-actions/%s' %
(server['id'], action['request_id'])).body['instanceAction']
events = action['events']
self.assertEqual(1, len(events), events)
event = events[0]
self.assertEqual('conductor_schedule_and_build_instances',
event['event'])
self.assertEqual('Error', event['result'])
# Normally non-admins cannot see the event traceback but we enabled
# that via policy in setUp so assert something was recorded.
self.assertIn('select_destinations', event['traceback'])
def delete_server_and_assert_cleanup(self, server):
"""Deletes the server and makes various cleanup checks.
- makes sure allocations from placement are gone
- makes sure the instance record is gone from both cells
- makes sure there are no leaked volume attachments
:param server: dict of the server resource to delete
"""
# Determine which cell the instance was in when the server was deleted
# in the API so we can check hard vs soft delete in the DB.
current_cell = self.host_to_cell_mappings[
server['OS-EXT-SRV-ATTR:host']]
# Delete the server and check that the allocations are gone from
# the placement service.
self._delete_and_check_allocations(server)
# Make sure the instance record is gone from both cell databases.
ctxt = nova_context.get_admin_context()
for cell_name in self.host_to_cell_mappings.values():
cell = self.cell_mappings[cell_name]
with nova_context.target_cell(ctxt, cell) as cctxt:
# If this is the current cell the instance was in when it was
# deleted it should be soft-deleted (instance.deleted!=0),
# otherwise it should be hard-deleted and getting it with a
# read_deleted='yes' context should still raise.
read_deleted = 'no' if current_cell == cell_name else 'yes'
with utils.temporary_mutation(
cctxt, read_deleted=read_deleted):
self.assertRaises(exception.InstanceNotFound,
objects.Instance.get_by_uuid,
cctxt, server['id'])
# Make sure there are no leaked volume attachments.
attachment_count = self._count_volume_attachments(server['id'])
self.assertEqual(0, attachment_count, 'Leaked volume attachments: %s' %
self.cinder.volume_to_attachment)
def test_boot_from_volume_with_isolated_image(self):
# Create our server without networking just to keep things simple.
image_id = nova_fixtures.CinderFixtureNewAttachFlow.IMAGE_BACKED_VOL
server_req_body = {
# There is no imageRef because this is boot from volume.
'server': {
'flavorRef': '1', # m1.tiny from DefaultFlavorsFixture,
'name': 'test_boot_from_volume_with_isolated_image',
'networks': 'none',
'block_device_mapping_v2': [{
'boot_index': 0,
'uuid': image_id,
'source_type': 'volume',
'destination_type': 'volume'
}]
}
}
# Note that we're using v2.1 by default but need v2.37 to use
# networks='none'.
with utils.temporary_mutation(self.api, microversion='2.37'):
server = self.api.post_server(server_req_body)
server = self._wait_for_state_change(self.api, server, 'ACTIVE')
# NOTE(mriedem): The instance is successfully scheduled but since
# the image_id from the volume_image_metadata isn't stored in the
# RequestSpec.image.id, and restrict_isolated_hosts_to_isolated_images
# is True, the isolated host (host1) is filtered out because the
# filter doesn't have enough information to know if the image within
# the volume can be used on that host.
self.assertEqual('host2', server['OS-EXT-SRV-ATTR:host'])
def test_boot_from_volume_with_isolated_image(self):
# Create our server without networking just to keep things simple.
image_id = nova_fixtures.CinderFixture.IMAGE_BACKED_VOL
server_req_body = {
# There is no imageRef because this is boot from volume.
'server': {
'flavorRef':
'1', # m1.tiny from DefaultFlavorsFixture,
'name':
'test_boot_from_volume_with_isolated_image',
'networks':
'none',
'block_device_mapping_v2': [{
'boot_index': 0,
'uuid': image_id,
'source_type': 'volume',
'destination_type': 'volume'
}]
}
}
# Note that we're using v2.1 by default but need v2.37 to use
# networks='none'.
with utils.temporary_mutation(self.api, microversion='2.37'):
server = self.api.post_server(server_req_body)
server = self._wait_for_state_change(self.api, server, 'ACTIVE')
# NOTE(mriedem): The instance is successfully scheduled but since
# the image_id from the volume_image_metadata isn't stored in the
# RequestSpec.image.id, and restrict_isolated_hosts_to_isolated_images
# is True, the isolated host (host1) is filtered out because the
# filter doesn't have enough information to know if the image within
# the volume can be used on that host.
self.assertEqual('host2', server['OS-EXT-SRV-ATTR:host'])
def test_update_available_resource_node_recreate(self):
# First we create a compute service to manage a couple of fake nodes.
compute = self.start_service('compute', 'node1')
# When start_service runs, it will create the node1 ComputeNode.
compute.manager.driver._set_nodes(['node1', 'node2'])
# Run the update_available_resource periodic to register node2.
ctxt = context.get_admin_context()
compute.manager.update_available_resource(ctxt)
# Make sure no compute nodes were orphaned or deleted.
self.assertNotIn('Deleting orphan compute node',
self.stdlog.logger.output)
# Now we should have two compute nodes, make sure the hypervisors API
# shows them.
hypervisors = self.api.api_get('/os-hypervisors').body['hypervisors']
self.assertEqual(2, len(hypervisors), hypervisors)
self.assertEqual({'node1', 'node2'},
set([hyp['hypervisor_hostname']
for hyp in hypervisors]))
# Now stub the driver to only report node1. This is making it look like
# node2 is no longer available when update_available_resource runs.
compute.manager.driver._nodes = ['node1']
ctxt = context.get_admin_context()
compute.manager.update_available_resource(ctxt)
# node2 should have been deleted, check the logs and API.
log = self.stdlog.logger.output
self.assertIn('Deleting orphan compute node', log)
self.assertIn('hypervisor host is node2', log)
hypervisors = self.api.api_get('/os-hypervisors').body['hypervisors']
self.assertEqual(1, len(hypervisors), hypervisors)
self.assertEqual('node1', hypervisors[0]['hypervisor_hostname'])
# But the node2 ComputeNode is still in the database with deleted!=0.
with utils.temporary_mutation(ctxt, read_deleted='yes'):
cn = objects.ComputeNode.get_by_host_and_nodename(
ctxt, 'node1', 'node2')
self.assertTrue(cn.deleted)
# Now stub the driver again to report node2 as being back and run
# the periodic task.
compute.manager.driver._nodes = ['node1', 'node2']
LOG.info('Running update_available_resource which should bring back '
'node2.')
compute.manager.update_available_resource(ctxt)
# The DBDuplicateEntry error should have been handled and resulted in
# updating the (soft) deleted record to no longer be deleted.
log = self.stdlog.logger.output
self.assertNotIn('DBDuplicateEntry', log)
# Should have two reported hypervisors again.
hypervisors = self.api.api_get('/os-hypervisors').body['hypervisors']
self.assertEqual(2, len(hypervisors), hypervisors)
# Now that the node2 record was un-soft-deleted, archiving should not
# remove any compute_nodes.
LOG.info('Archiving the database.')
archived = db_api.archive_deleted_rows(1000)[0]
self.assertNotIn('compute_nodes', archived)
cn2 = objects.ComputeNode.get_by_host_and_nodename(
ctxt, 'node1', 'node2')
self.assertFalse(cn2.deleted)
self.assertIsNone(cn2.deleted_at)
# The node2 id and uuid should not have changed in the DB.
self.assertEqual(cn.id, cn2.id)
self.assertEqual(cn.uuid, cn2.uuid)
def assert_hypervisor_usage(
self,
compute_node_uuid,
flavor,
volume_backed,
):
"""Asserts the given hypervisor's resource usage matches the
given flavor (assumes a single instance on the hypervisor).
:param compute_node_uuid: UUID of the ComputeNode to check.
:param flavor: "flavor" entry dict from from GET /flavors/{flavor_id}
:param volume_backed: True if the flavor is used with a volume-backed
server, False otherwise.
"""
# GET /os-hypervisors/{uuid} requires at least 2.53
with utils.temporary_mutation(self.admin_api, microversion='2.53'):
hypervisor = self.admin_api.api_get(
'/os-hypervisors/%s' % compute_node_uuid).body['hypervisor']
if volume_backed:
expected_disk_usage = 0
else:
expected_disk_usage = flavor['disk']
# Account for reserved_host_disk_mb.
expected_disk_usage += compute_utils.convert_mb_to_ceil_gb(
CONF.reserved_host_disk_mb)
self.assertEqual(expected_disk_usage, hypervisor['local_gb_used'])
# Account for reserved_host_memory_mb.
expected_ram_usage = CONF.reserved_host_memory_mb + flavor['ram']
self.assertEqual(expected_ram_usage, hypervisor['memory_mb_used'])
# Account for reserved_host_cpus.
expected_vcpu_usage = CONF.reserved_host_cpus + flavor['vcpus']
self.assertEqual(expected_vcpu_usage, hypervisor['vcpus_used'])
def assert_hypervisor_usage(self, compute_node_uuid, flavor,
volume_backed):
"""Asserts the given hypervisor's resource usage matches the
given flavor (assumes a single instance on the hypervisor).
:param compute_node_uuid: UUID of the ComputeNode to check.
:param flavor: "flavor" entry dict from from GET /flavors/{flavor_id}
:param volume_backed: True if the flavor is used with a volume-backed
server, False otherwise.
"""
# GET /os-hypervisors/{uuid} requires at least 2.53
with utils.temporary_mutation(self.admin_api, microversion='2.53'):
hypervisor = self.admin_api.api_get(
'/os-hypervisors/%s' % compute_node_uuid).body['hypervisor']
if volume_backed:
expected_disk_usage = 0
else:
expected_disk_usage = flavor['disk']
# Account for reserved_host_disk_mb.
expected_disk_usage += compute_utils.convert_mb_to_ceil_gb(
CONF.reserved_host_disk_mb)
self.assertEqual(expected_disk_usage, hypervisor['local_gb_used'])
# Account for reserved_host_memory_mb.
expected_ram_usage = CONF.reserved_host_memory_mb + flavor['ram']
self.assertEqual(expected_ram_usage, hypervisor['memory_mb_used'])
# Account for reserved_host_cpus.
expected_vcpu_usage = CONF.reserved_host_cpus + flavor['vcpus']
self.assertEqual(expected_vcpu_usage, hypervisor['vcpus_used'])
def _delete_and_check_allocations(self, server):
"""Delete the instance and asserts that the allocations are cleaned
If the server was moved (resized or live migrated), also checks that
migration-based allocations are also cleaned up.
:param server: The API representation of the instance to be deleted
:returns: The uuid of the migration record associated with the resize
or cold migrate operation
"""
# First check to see if there is a related migration record so we can
# assert its allocations (if any) are not leaked.
with utils.temporary_mutation(self.admin_api, microversion='2.59'):
migrations = self.admin_api.api_get(
'/os-migrations?instance_uuid=%s' %
server['id']).body['migrations']
if migrations:
# If there is more than one migration, they are sorted by
# created_at in descending order so we'll get the last one
# which is probably what we'd always want anyway.
migration_uuid = migrations[0]['uuid']
else:
migration_uuid = None
self._delete_server(server)
# NOTE(gibi): The resource allocation is deleted after the instance is
# destroyed in the db so wait_until_deleted might return before the
# the resource are deleted in placement. So we need to wait for the
# instance.delete.end notification as that is emitted after the
# resources are freed.
fake_notifier.wait_for_versioned_notifications('instance.delete.end')
for rp_uuid in [
self._get_provider_uuid_by_host(hostname)
for hostname in self.computes.keys()
]:
self.assertRequestMatchesUsage(
{
'VCPU': 0,
'MEMORY_MB': 0,
'DISK_GB': 0
}, rp_uuid)
# and no allocations for the deleted server
allocations = self._get_allocations_by_server_uuid(server['id'])
self.assertEqual(0, len(allocations))
if migration_uuid:
# and no allocations for the delete migration
allocations = self._get_allocations_by_server_uuid(migration_uuid)
self.assertEqual(0, len(allocations))
return migration_uuid
def test_init_application_passes_sys_argv_to_config(self):
with utils.temporary_mutation(sys, argv=mock.sentinel.argv):
with mock.patch('nova.config.parse_args') as mock_parse_args:
wsgi_app.init_application('test-app')
mock_parse_args.assert_called_once_with(
mock.sentinel.argv,
default_config_files=[
'/etc/nova/api-paste.ini', '/etc/nova/nova.conf'
])
def instance_update_at_top(self, message, instance, **kwargs):
"""Update an instance in the DB if we're a top level cell."""
if not self._at_the_top():
return
instance_uuid = instance['uuid']
# Remove things that we can't update in the top level cells.
# 'metadata' is only updated in the API cell, so don't overwrite
# it based on what child cells say. Make sure to update
# 'cell_name' based on the routing path.
items_to_remove = [
'id', 'security_groups', 'instance_type', 'volumes', 'cell_name',
'name', 'metadata'
]
for key in items_to_remove:
instance.pop(key, None)
instance['cell_name'] = _reverse_path(message.routing_path)
# Fixup info_cache. We'll have to update this separately if
# it exists.
info_cache = instance.pop('info_cache', None)
if info_cache is not None:
info_cache.pop('id', None)
info_cache.pop('instance', None)
# Fixup system_metadata (should be a dict for update, not a list)
if ('system_metadata' in instance
and isinstance(instance['system_metadata'], list)):
sys_metadata = dict([(md['key'], md['value'])
for md in instance['system_metadata']])
instance['system_metadata'] = sys_metadata
LOG.debug(
_("Got update for instance %(instance_uuid)s: "
"%(instance)s") % locals())
# It's possible due to some weird condition that the instance
# was already set as deleted... so we'll attempt to update
# it with permissions that allows us to read deleted.
with utils.temporary_mutation(message.ctxt, read_deleted="yes"):
try:
self.db.instance_update(message.ctxt,
instance_uuid,
instance,
update_cells=False)
except exception.NotFound:
# FIXME(comstud): Strange. Need to handle quotas here,
# if we actually want this code to remain..
self.db.instance_create(message.ctxt, instance)
if info_cache:
self.db.instance_info_cache_update(message.ctxt,
instance_uuid,
info_cache,
update_cells=False)
def _get_compute_node_id_by_host(self, host):
# we specifically need the integer id of the node not the UUID so we
# need to use the old microversion
with utils.temporary_mutation(self.admin_api, microversion='2.52'):
hypers = self.admin_api.api_get(
'os-hypervisors').body['hypervisors']
for hyper in hypers:
if hyper['hypervisor_hostname'] == host:
return hyper['id']
self.fail('Hypervisor with hostname=%s not found' % host)
def _test_host_status_unknown_only(self, func_name, *args):
admin_func = getattr(self.admin_api, func_name)
func = getattr(self.api, func_name)
# Run the operation as admin and extract the server from the response.
server = self._get_server(admin_func(*args))
# We need to wait for ACTIVE if this was a post rebuild server action,
# else a subsequent rebuild request will fail with a 409 in the API.
self._wait_for_state_change(server, 'ACTIVE')
# Verify admin can see the host status UP.
self.assertEqual('UP', server['host_status'])
# Get server as normal non-admin user.
server = self._get_server(func(*args))
self._wait_for_state_change(server, 'ACTIVE')
# Verify non-admin do not receive the host_status field because it is
# not UNKNOWN.
self.assertNotIn('host_status', server)
# Stop the compute service to trigger UNKNOWN host_status.
self.compute.stop()
# Advance time by 30 minutes so nova considers service as down.
minutes_from_now = timeutils.utcnow() + datetime.timedelta(minutes=30)
timeutils.set_time_override(override_time=minutes_from_now)
self.addCleanup(timeutils.clear_time_override)
# Run the operation as admin and extract the server from the response.
server = self._get_server(admin_func(*args))
# Verify admin can see the host status UNKNOWN.
self.assertEqual('UNKNOWN', server['host_status'])
# Now that the compute service is down, the rebuild will not ever
# complete. But we're only interested in what would be returned from
# the API post rebuild action, so reset the state to ACTIVE to allow
# the next rebuild request to go through without a 409 error.
self._set_server_state_active(server)
# Run the operation as a normal non-admin user and extract the server
# from the response.
server = self._get_server(func(*args))
# Verify non-admin can see the host status UNKNOWN too.
self.assertEqual('UNKNOWN', server['host_status'])
self._set_server_state_active(server)
# Now, adjust the policy to make it so only admin are allowed to see
# UNKNOWN host status only.
self.policy.set_rules(
{self.host_status_unknown_only_rule: 'rule:admin_api'},
overwrite=False)
# Run the operation as a normal non-admin user and extract the server
# from the response.
server = self._get_server(func(*args))
# Verify non-admin do not receive the host_status field.
self.assertNotIn('host_status', server)
self._set_server_state_active(server)
# Verify that admin will not receive ths host_status field if the
# API microversion < 2.16.
with utils.temporary_mutation(self.admin_api, microversion='2.15'):
server = self._get_server(admin_func(*args))
self.assertNotIn('host_status', server)
def _build_server(self,
name=None,
image_uuid=None,
flavor_id=None,
networks=None,
az=None,
host=None):
"""Build a request for the server create API.
:param name: A name for the server.
:param image_uuid: The ID of an existing image.
:param flavor_id: The ID of an existing flavor.
:param networks: A dict of networks to attach or a string of 'none' or
'auto'.
:param az: The name of the availability zone the instance should
request.
:param host: The host to boot the instance on. Requires API
microversion 2.74 or greater.
:returns: The generated request body.
"""
if not name:
name = ''.join(
random.choice(string.ascii_lowercase) for i in range(20))
if image_uuid is None: # we need to handle ''
# NOTE(takashin): In API version 2.36, image APIs were deprecated.
# In API version 2.36 or greater, self.api.get_images() returns
# a 404 error. In that case, 'image_uuid' should be specified.
with utils.temporary_mutation(self.api, microversion='2.35'):
image_uuid = self.api.get_images()[0]['id']
if not flavor_id:
# Set a valid flavorId
flavor_id = self.api.get_flavors()[0]['id']
server = {
'name': name,
'imageRef': image_uuid,
'flavorRef': 'http://fake.server/%s' % flavor_id,
}
if networks is not None:
server['networks'] = networks
if az is not None:
server['availability_zone'] = az
# This requires at least microversion 2.74 to work
if host is not None:
server['host'] = host
return server
def notify_usage_exists(context, instance_ref, current_period=False,
ignore_missing_network_data=True,
system_metadata=None, extra_usage_info=None):
"""Generates 'exists' notification for an instance for usage auditing
purposes.
:param current_period: if True, this will generate a usage for the
current usage period; if False, this will generate a usage for the
previous audit period.
:param ignore_missing_network_data: if True, log any exceptions generated
while getting network info; if False, raise the exception.
:param system_metadata: system_metadata DB entries for the instance,
if not None. *NOTE*: Currently unused here in trunk, but needed for
potential custom modifications.
:param extra_usage_info: Dictionary containing extra values to add or
override in the notification if not None.
"""
audit_start, audit_end = notifications.audit_period_bounds(current_period)
bw = notifications.bandwidth_usage(instance_ref, audit_start,
ignore_missing_network_data)
if system_metadata is None:
try:
if instance_ref.get('deleted'):
with utils.temporary_mutation(context, read_deleted='yes'):
system_metadata = db.instance_system_metadata_get(
context, instance_ref.uuid)
else:
system_metadata = db.instance_system_metadata_get(
context, instance_ref.uuid)
except exception.NotFound:
system_metadata = {}
# add image metadata to the notification:
image_meta = notifications.image_meta(system_metadata)
extra_info = dict(audit_period_beginning=str(audit_start),
audit_period_ending=str(audit_end),
bandwidth=bw, image_meta=image_meta)
if extra_usage_info:
extra_info.update(extra_usage_info)
notify_about_instance_usage(context, instance_ref, 'exists',
system_metadata=system_metadata, extra_usage_info=extra_info)
def instance_update_at_top(self, message, instance, **kwargs):
"""Update an instance in the DB if we're a top level cell."""
if not self._at_the_top():
return
instance_uuid = instance['uuid']
# Remove things that we can't update in the top level cells.
# 'metadata' is only updated in the API cell, so don't overwrite
# it based on what child cells say. Make sure to update
# 'cell_name' based on the routing path.
items_to_remove = ['id', 'security_groups', 'instance_type',
'volumes', 'cell_name', 'name', 'metadata']
for key in items_to_remove:
instance.pop(key, None)
instance['cell_name'] = _reverse_path(message.routing_path)
# Fixup info_cache. We'll have to update this separately if
# it exists.
info_cache = instance.pop('info_cache', None)
if info_cache is not None:
info_cache.pop('id', None)
info_cache.pop('instance', None)
# Fixup system_metadata (should be a dict for update, not a list)
if ('system_metadata' in instance and
isinstance(instance['system_metadata'], list)):
sys_metadata = dict([(md['key'], md['value'])
for md in instance['system_metadata']])
instance['system_metadata'] = sys_metadata
LOG.debug(_("Got update for instance %(instance_uuid)s: "
"%(instance)s") % locals())
# It's possible due to some weird condition that the instance
# was already set as deleted... so we'll attempt to update
# it with permissions that allows us to read deleted.
with utils.temporary_mutation(message.ctxt, read_deleted="yes"):
try:
self.db.instance_update(message.ctxt, instance_uuid,
instance, update_cells=False)
except exception.NotFound:
# FIXME(comstud): Strange. Need to handle quotas here,
# if we actually want this code to remain..
self.db.instance_create(message.ctxt, instance)
if info_cache:
self.db.instance_info_cache_update(message.ctxt, instance_uuid,
info_cache, update_cells=False)
def test_rebuild_instance_with_image_traits_on_shared_rp(self):
shared_rp_uuid = self.create_shared_storage_rp()
# add both cn_rp and shared_rp under one aggregate
self._set_aggregate(shared_rp_uuid, uuids.shr_disk_agg)
self._set_aggregate(self.host_uuid, uuids.shr_disk_agg)
self.assertIn("DISK_GB", self._get_provider_inventory(self.host_uuid))
# run update_available_resource periodic task after configuring shared
# resource provider to update compute node resources
self._run_periodics()
# we expect that the virt driver stops reporting DISK_GB on the compute
# RP as soon as a shared RP with DISK_GB is created in the compute tree
self.assertNotIn("DISK_GB",
self._get_provider_inventory(self.host_uuid))
server = self._create_server(
image_uuid='155d900f-4e14-4e4c-a73d-069cbf4541e6',
flavor_id=1,
networks='none')
rebuild_image_ref = self.glance.auto_disk_config_enabled_image['id']
with utils.temporary_mutation(self.api, microversion='2.35'):
self.api.api_put(
'/images/%s/metadata' % rebuild_image_ref,
{'metadata': {
'trait:STORAGE_DISK_SSD': 'required'
}})
rebuild_req_body = {'rebuild': {'imageRef': rebuild_image_ref}}
self.api.api_post('/servers/%s/action' % server['id'],
rebuild_req_body)
self._wait_for_server_parameter(server,
{'OS-EXT-STS:task_state': None})
# get shared_rp and cn_rp usages
shared_rp_usages = self._get_provider_usages(shared_rp_uuid)
cn_rp_usages = self._get_provider_usages(self.host_uuid)
# Check if DISK_GB resource is allocated from shared_RP and the
# remaining resources are allocated from host_uuid.
self.assertEqual({'DISK_GB': 1}, shared_rp_usages)
self.assertEqual({'MEMORY_MB': 512, 'VCPU': 1}, cn_rp_usages)
allocs = self._get_allocations_by_server_uuid(server['id'])
self.assertIn(self.host_uuid, allocs)
server = self.api.get_server(server['id'])
self.assertEqual(rebuild_image_ref, server['image']['id'])
def _delete_and_check_allocations(self, server):
"""Delete the instance and asserts that the allocations are cleaned
If the server was moved (resized or live migrated), also checks that
migration-based allocations are also cleaned up.
:param server: The API representation of the instance to be deleted
"""
# First check to see if there is a related migration record so we can
# assert its allocations (if any) are not leaked.
with utils.temporary_mutation(self.admin_api, microversion='2.59'):
migrations = self.admin_api.api_get(
'/os-migrations?instance_uuid=%s' %
server['id']).body['migrations']
if migrations:
# If there is more than one migration, they are sorted by
# created_at in descending order so we'll get the last one
# which is probably what we'd always want anyway.
migration_uuid = migrations[0]['uuid']
else:
migration_uuid = None
self.api.delete_server(server['id'])
self._wait_until_deleted(server)
# NOTE(gibi): The resource allocation is deleted after the instance is
# destroyed in the db so wait_until_deleted might return before the
# the resource are deleted in placement. So we need to wait for the
# instance.delete.end notification as that is emitted after the
# resources are freed.
fake_notifier.wait_for_versioned_notifications('instance.delete.end')
for rp_uuid in [self._get_provider_uuid_by_host(hostname)
for hostname in self.computes.keys()]:
self.assertRequestMatchesUsage({'VCPU': 0,
'MEMORY_MB': 0,
'DISK_GB': 0}, rp_uuid)
# and no allocations for the deleted server
allocations = self._get_allocations_by_server_uuid(server['id'])
self.assertEqual(0, len(allocations))
if migration_uuid:
# and no allocations for the delete migration
allocations = self._get_allocations_by_server_uuid(migration_uuid)
self.assertEqual(0, len(allocations))
def test_rebuild_instance_with_image_traits_on_shared_rp_no_valid_host(
self):
shared_rp_uuid = self.create_shared_storage_rp()
# add both cn_rp and shared_rp under one aggregate
self._set_aggregate(shared_rp_uuid, uuids.shr_disk_agg)
self._set_aggregate(self.host_uuid, uuids.shr_disk_agg)
self.assertIn("DISK_GB", self._get_provider_inventory(self.host_uuid))
# run update_available_resource periodic task after configuring shared
# resource provider to update compute node resources
self._run_periodics()
# we expect that the virt driver stops reporting DISK_GB on the compute
# RP as soon as a shared RP with DISK_GB is created in the compute tree
self.assertNotIn("DISK_GB",
self._get_provider_inventory(self.host_uuid))
# create server
org_image_id = '155d900f-4e14-4e4c-a73d-069cbf4541e6'
server = self._create_server(
image_uuid=org_image_id,
flavor_id=1,
networks='none',
)
rebuild_image_ref = self.glance.auto_disk_config_enabled_image['id']
with utils.temporary_mutation(self.api, microversion='2.35'):
self.api.api_put('/images/%s/metadata' % rebuild_image_ref,
{'metadata': {
'trait:CUSTOM_FOO': 'required'
}})
rebuild_req_body = {'rebuild': {'imageRef': rebuild_image_ref}}
self.api.api_post('/servers/%s/action' % server['id'],
rebuild_req_body)
# Look for the failed rebuild action.
self._wait_for_action_fail_completion(server, instance_actions.REBUILD,
'rebuild_server')
# Assert the server image_ref was rolled back on failure.
server = self.api.get_server(server['id'])
self.assertEqual(org_image_id, server['image']['id'])
# The server should be in ERROR state
self.assertEqual('ERROR', server['status'])
self.assertIn('No valid host', server['fault']['message'])
def instance_update_at_top(self, message, instance, **kwargs):
"""Update an instance in the DB if we're a top level cell."""
if not self._at_the_top():
return
instance_uuid = instance["uuid"]
routing_path = message.routing_path
instance["cell_name"] = _reverse_path(routing_path)
# Remove things that we can't update in the top level cells.
# 'cell_name' is included in this list.. because we'll set it
# ourselves based on the reverse of the routing path. metadata
# is only updated in the API cell, so we don't listen to what
# the child cell tells us.
items_to_remove = ["id", "security_groups", "instance_type", "volumes", "cell_name", "name", "metadata"]
for key in items_to_remove:
instance.pop(key, None)
# Fixup info_cache. We'll have to update this separately if
# it exists.
info_cache = instance.pop("info_cache", None)
if info_cache is not None:
info_cache.pop("id", None)
info_cache.pop("instance", None)
# Fixup system_metadata (should be a dict for update, not a list)
if "system_metadata" in instance and isinstance(instance["system_metadata"], list):
sys_metadata = dict([(md["key"], md["value"]) for md in instance["system_metadata"]])
instance["system_metadata"] = sys_metadata
LOG.debug(_("Got update for instance %(instance_uuid)s: " "%(instance)s") % locals())
# It's possible due to some weird condition that the instance
# was already set as deleted... so we'll attempt to update
# it with permissions that allows us to read deleted.
with utils.temporary_mutation(message.ctxt, read_deleted="yes"):
try:
self.db.instance_update(message.ctxt, instance_uuid, instance, update_cells=False)
except exception.NotFound:
# FIXME(comstud): Strange. Need to handle quotas here,
# if we actually want this code to remain..
self.db.instance_create(message.ctxt, instance)
if info_cache:
self.db.instance_info_cache_update(message.ctxt, instance_uuid, info_cache, update_cells=False)
def _build_server(self, flavor_id, image=None):
server = {}
if image is None:
# TODO(stephenfin): We need to stop relying on this API
with utils.temporary_mutation(self.api, microversion='2.35'):
image = self.api.get_images()[0]
LOG.debug("Image: %s", image)
# We now have a valid imageId
server[self._image_ref_parameter] = image['id']
else:
server[self._image_ref_parameter] = image
# Set a valid flavorId
flavor = self.api.get_flavor(flavor_id)
LOG.debug("Using flavor: %s", flavor)
server[self._flavor_ref_parameter] = ('http://fake.server/%s'
% flavor['id'])
# Set a valid server name
server_name = self.get_unused_server_name()
server['name'] = server_name
return server
def test_serial_no_valid_host_then_pass_with_third_host(self):
"""Creates 2 servers in order (not a multi-create request) in an
anti-affinity group so there will be 1 server on each host. Then
attempts to live migrate the first server which will fail because the
only other available host will be full. Then starts up a 3rd compute
service and retries the live migration which should then pass.
"""
# Create the anti-affinity group used for the servers.
group = self.api.post_server_groups(
{'name': 'test_serial_no_valid_host_then_pass_with_third_host',
'policies': ['anti-affinity']})
servers = []
for x in range(2):
server = self._build_minimal_create_server_request(
self.api,
'test_serial_no_valid_host_then_pass_with_third_host-%d' % x,
networks='none')
# Add the group hint so the server is created in our group.
server_req = {
'server': server,
'os:scheduler_hints': {'group': group['id']}
}
# Use microversion 2.37 for passing networks='none'.
with utils.temporary_mutation(self.api, microversion='2.37'):
server = self.api.post_server(server_req)
servers.append(
self._wait_for_state_change(
self.admin_api, server, 'ACTIVE'))
# Make sure each server is on a unique host.
hosts = set([svr['OS-EXT-SRV-ATTR:host'] for svr in servers])
self.assertEqual(2, len(hosts))
# And make sure the group has 2 members.
members = self.api.get_server_group(group['id'])['members']
self.assertEqual(2, len(members))
# Now attempt to live migrate one of the servers which should fail
# because we don't have a free host. Since we're using microversion 2.1
# the scheduling will be synchronous and we should get back a 400
# response for the NoValidHost error.
body = {
'os-migrateLive': {
'host': None,
'block_migration': False,
'disk_over_commit': False
}
}
# Specifically use the first server since that was the first member
# added to the group.
server = servers[0]
ex = self.assertRaises(client.OpenStackApiException,
self.admin_api.post_server_action,
server['id'], body)
self.assertEqual(400, ex.response.status_code)
self.assertIn('No valid host', six.text_type(ex))
# Now start up a 3rd compute service and retry the live migration which
# should work this time.
self.start_service('compute', host='host3')
self.admin_api.post_server_action(server['id'], body)
server = self._wait_for_state_change(self.admin_api, server, 'ACTIVE')
# Now the server should be on host3 since that was the only available
# host for the live migration.
self.assertEqual('host3', server['OS-EXT-SRV-ATTR:host'])
def _set_forced_down(self, service, forced_down):
# Use microversion 2.53 for PUT /os-services/{service_id} force down.
with utils.temporary_mutation(self.admin_api, microversion='2.53'):
self.admin_api.put_service_force_down(service.service_ref.uuid,
forced_down)
def test_serial_no_valid_host_then_pass_with_third_host(self):
"""Creates 2 servers in order (not a multi-create request) in an
anti-affinity group so there will be 1 server on each host. Then
attempts to live migrate the first server which will fail because the
only other available host will be full. Then starts up a 3rd compute
service and retries the live migration which should then pass.
"""
# Create the anti-affinity group used for the servers.
group = self.api.post_server_groups(
{'name': 'test_serial_no_valid_host_then_pass_with_third_host',
'policies': ['anti-affinity']})
servers = []
for x in range(2):
server = self._build_minimal_create_server_request(
self.api,
'test_serial_no_valid_host_then_pass_with_third_host-%d' % x,
networks='none')
# Add the group hint so the server is created in our group.
server_req = {
'server': server,
'os:scheduler_hints': {'group': group['id']}
}
# Use microversion 2.37 for passing networks='none'.
with utils.temporary_mutation(self.api, microversion='2.37'):
server = self.api.post_server(server_req)
servers.append(
self._wait_for_state_change(
self.admin_api, server, 'ACTIVE'))
# Make sure each server is on a unique host.
hosts = set([svr['OS-EXT-SRV-ATTR:host'] for svr in servers])
self.assertEqual(2, len(hosts))
# And make sure the group has 2 members.
members = self.api.get_server_group(group['id'])['members']
self.assertEqual(2, len(members))
# Now attempt to live migrate one of the servers which should fail
# because we don't have a free host. Since we're using microversion 2.1
# the scheduling will be synchronous and we should get back a 400
# response for the NoValidHost error.
body = {
'os-migrateLive': {
'host': None,
'block_migration': False,
'disk_over_commit': False
}
}
# Specifically use the first server since that was the first member
# added to the group.
server = servers[0]
ex = self.assertRaises(client.OpenStackApiException,
self.admin_api.post_server_action,
server['id'], body)
self.assertEqual(400, ex.response.status_code)
self.assertIn('No valid host', six.text_type(ex))
# Now start up a 3rd compute service and retry the live migration which
# should work this time.
fake.set_nodes(['host3'])
self.start_service('compute', host='host3')
self.admin_api.post_server_action(server['id'], body)
server = self._wait_for_state_change(self.admin_api, server, 'ACTIVE')
# Now the server should be on host3 since that was the only available
# host for the live migration.
self.assertEqual('host3', server['OS-EXT-SRV-ATTR:host'])
def usage_from_instance(context, instance_ref, network_info,
system_metadata, **kw):
"""Get usage information for an instance which is common to all
notifications.
:param network_info: network_info provided if not None
:param system_metadata: system_metadata DB entries for the instance,
if not None. *NOTE*: Currently unused here in trunk, but needed for
potential custom modifications.
"""
def null_safe_str(s):
return str(s) if s else ''
image_ref_url = utils.generate_image_url(instance_ref['image_ref'])
instance_type_name = instance_ref.get('instance_type', {}).get('name', '')
if system_metadata is None:
try:
if instance_ref.get('deleted'):
with utils.temporary_mutation(context, read_deleted='yes'):
system_metadata = db.instance_system_metadata_get(
context, instance_ref['uuid'])
else:
system_metadata = db.instance_system_metadata_get(
context, instance_ref['uuid'])
except exception.NotFound:
system_metadata = {}
usage_info = dict(
# Owner properties
tenant_id=instance_ref['project_id'],
user_id=instance_ref['user_id'],
# Identity properties
instance_id=instance_ref['uuid'],
display_name=instance_ref['display_name'],
reservation_id=instance_ref['reservation_id'],
# Type properties
instance_type=instance_type_name,
instance_type_id=instance_ref['instance_type_id'],
architecture=instance_ref['architecture'],
# Capacity properties
memory_mb=instance_ref['memory_mb'],
disk_gb=instance_ref['root_gb'] + instance_ref['ephemeral_gb'],
vcpus=instance_ref['vcpus'],
# Note(dhellmann): This makes the disk_gb value redundant, but
# we are keeping it for backwards-compatibility with existing
# users of notifications.
root_gb=instance_ref['root_gb'],
ephemeral_gb=instance_ref['ephemeral_gb'],
# Location properties
host=instance_ref['host'],
availability_zone=instance_ref['availability_zone'],
# Date properties
created_at=str(instance_ref['created_at']),
# Nova's deleted vs terminated instance terminology is confusing,
# this should be when the instance was deleted (i.e. terminated_at),
# not when the db record was deleted. (mdragon)
deleted_at=null_safe_str(instance_ref.get('terminated_at')),
launched_at=null_safe_str(instance_ref.get('launched_at')),
# Image properties
image_ref_url=image_ref_url,
os_type=instance_ref['os_type'],
kernel_id=instance_ref['kernel_id'],
ramdisk_id=instance_ref['ramdisk_id'],
# Status properties
state=instance_ref['vm_state'],
state_description=null_safe_str(instance_ref.get('task_state')),
)
if network_info is not None:
usage_info['fixed_ips'] = network_info.fixed_ips()
# add image metadata
image_meta_props = image_meta(system_metadata)
usage_info["image_meta"] = image_meta_props
usage_info.update(kw)
return usage_info
def _resize_and_validate(self,
volume_backed=False,
stopped=False,
target_host=None):
"""Creates and resizes the server to another cell. Validates various
aspects of the server and its related records (allocations, migrations,
actions, VIF tags, etc).
:param volume_backed: True if the server should be volume-backed, False
if image-backed.
:param stopped: True if the server should be stopped prior to resize,
False if the server should be ACTIVE
:param target_host: If not None, triggers a cold migration to the
specified host.
:returns: tuple of:
- server response object
- source compute node resource provider uuid
- target compute node resource provider uuid
- old flavor
- new flavor
"""
# Create the server.
flavors = self.api.get_flavors()
old_flavor = flavors[0]
server = self._create_server(old_flavor, volume_backed=volume_backed)
original_host = server['OS-EXT-SRV-ATTR:host']
image_uuid = None if volume_backed else server['image']['id']
# Our HostNameWeigher ensures the server starts in cell1, so we expect
# the server AZ to be cell1 as well.
self.assertEqual('cell1', server['OS-EXT-AZ:availability_zone'])
if stopped:
# Stop the server before resizing it.
self.api.post_server_action(server['id'], {'os-stop': None})
self._wait_for_state_change(self.api, server, 'SHUTOFF')
# Before resizing make sure quota usage is only 1 for total instances.
self.assert_quota_usage(expected_num_instances=1)
if target_host:
# Cold migrate the server to the target host.
new_flavor = old_flavor # flavor does not change for cold migrate
body = {'migrate': {'host': target_host}}
expected_host = target_host
else:
# Resize it which should migrate the server to the host in the
# other cell.
new_flavor = flavors[1]
body = {'resize': {'flavorRef': new_flavor['id']}}
expected_host = 'host1' if original_host == 'host2' else 'host2'
self.stub_image_create()
self.api.post_server_action(server['id'], body)
# Wait for the server to be resized and then verify the host has
# changed to be the host in the other cell.
server = self._wait_for_state_change(self.api, server, 'VERIFY_RESIZE')
self.assertEqual(expected_host, server['OS-EXT-SRV-ATTR:host'])
# Assert that the instance is only listed one time from the API (to
# make sure it's not listed out of both cells).
# Note that we only get one because the DB API excludes hidden
# instances by default (see instance_get_all_by_filters_sort).
servers = self.api.get_servers()
self.assertEqual(1, len(servers),
'Unexpected number of servers: %s' % servers)
self.assertEqual(expected_host, servers[0]['OS-EXT-SRV-ATTR:host'])
# And that there is only one migration record.
migrations = self.api.api_get('/os-migrations?instance_uuid=%s' %
server['id']).body['migrations']
self.assertEqual(
1, len(migrations),
'Unexpected number of migrations records: %s' % migrations)
migration = migrations[0]
self.assertEqual('finished', migration['status'])
# There should be at least two actions, one for create and one for the
# resize. There will be a third action if the server was stopped.
actions = self.api.api_get('/servers/%s/os-instance-actions' %
server['id']).body['instanceActions']
expected_num_of_actions = 3 if stopped else 2
self.assertEqual(expected_num_of_actions, len(actions), actions)
# Each action should have events (make sure these were copied from
# the source cell to the target cell).
for action in actions:
detail = self.api.api_get(
'/servers/%s/os-instance-actions/%s' %
(server['id'], action['request_id'])).body['instanceAction']
self.assertNotEqual(0, len(detail['events']), detail)
# The tag should still be present on the server.
self.assertEqual(1, len(server['tags']),
'Server tags not found in target cell.')
self.assertEqual('test', server['tags'][0])
# Confirm the source node has allocations for the old flavor and the
# target node has allocations for the new flavor.
source_rp_uuid = self._get_provider_uuid_by_host(original_host)
# The source node allocations should be on the migration record.
source_allocations = self._get_allocations_by_provider_uuid(
source_rp_uuid)[migration['uuid']]['resources']
self.assertFlavorMatchesAllocation(old_flavor,
source_allocations,
volume_backed=volume_backed)
target_rp_uuid = self._get_provider_uuid_by_host(expected_host)
# The target node allocations should be on the instance record.
target_allocations = self._get_allocations_by_provider_uuid(
target_rp_uuid)[server['id']]['resources']
self.assertFlavorMatchesAllocation(new_flavor,
target_allocations,
volume_backed=volume_backed)
# The instance, in the target cell DB, should have the old and new
# flavor stored with it with the values we expect at this point.
target_cell_name = self.host_to_cell_mappings[expected_host]
self.assertEqual(target_cell_name,
server['OS-EXT-AZ:availability_zone'])
target_cell = self.cell_mappings[target_cell_name]
admin_context = nova_context.get_admin_context()
with nova_context.target_cell(admin_context, target_cell) as cctxt:
inst = objects.Instance.get_by_uuid(cctxt,
server['id'],
expected_attrs=['flavor'])
self.assertIsNotNone(
inst.old_flavor,
'instance.old_flavor not saved in target cell')
self.assertIsNotNone(
inst.new_flavor,
'instance.new_flavor not saved in target cell')
self.assertEqual(inst.flavor.flavorid, inst.new_flavor.flavorid)
if target_host: # cold migrate so flavor does not change
self.assertEqual(inst.flavor.flavorid,
inst.old_flavor.flavorid)
else:
self.assertNotEqual(inst.flavor.flavorid,
inst.old_flavor.flavorid)
self.assertEqual(old_flavor['id'], inst.old_flavor.flavorid)
self.assertEqual(new_flavor['id'], inst.new_flavor.flavorid)
# Assert the ComputeManager._set_instance_info fields
# are correct after the resize.
self.assert_instance_fields_match_flavor(inst, new_flavor)
# The availability_zone field in the DB should also be updated.
self.assertEqual(target_cell_name, inst.availability_zone)
# Assert the VIF tag was carried through to the target cell DB.
interface_attachments = self.api.get_port_interfaces(server['id'])
self.assertEqual(1, len(interface_attachments))
self.assertEqual('private', interface_attachments[0]['tag'])
if volume_backed:
# Assert the BDM tag was carried through to the target cell DB.
volume_attachments = self.api.get_server_volumes(server['id'])
self.assertEqual(1, len(volume_attachments))
self.assertEqual('root', volume_attachments[0]['tag'])
# Make sure the guest is no longer tracked on the source node.
source_guest_uuids = (
self.computes[original_host].manager.driver.list_instance_uuids())
self.assertNotIn(server['id'], source_guest_uuids)
# And the guest is on the target node hypervisor.
target_guest_uuids = (
self.computes[expected_host].manager.driver.list_instance_uuids())
self.assertIn(server['id'], target_guest_uuids)
# The source hypervisor continues to report usage in the hypervisors
# API because even though the guest was destroyed there, the instance
# resources are still claimed on that node in case the user reverts.
self.assert_hypervisor_usage(source_rp_uuid, old_flavor, volume_backed)
# The new flavor should show up with resource usage on the target host.
self.assert_hypervisor_usage(target_rp_uuid, new_flavor, volume_backed)
# While we have a copy of the instance in each cell database make sure
# that quota usage is only reporting 1 (because one is hidden).
self.assert_quota_usage(expected_num_instances=1)
# For a volume-backed server, at this point there should be two volume
# attachments for the instance: one tracked in the source cell and
# one in the target cell.
if volume_backed:
self.assertEqual(2, self._count_volume_attachments(server['id']),
self.cinder.volume_to_attachment)
# Assert the expected power state.
expected_power_state = 4 if stopped else 1
self.assertEqual(expected_power_state,
server['OS-EXT-STS:power_state'],
"Unexpected power state after resize.")
# For an image-backed server, a snapshot image should have been created
# and then deleted during the resize.
if volume_backed:
self.assertEqual('', server['image'])
self.assertEqual(
0, len(self.created_images),
"Unexpected image create during volume-backed resize")
else:
# The original image for the server shown in the API should not
# have changed even if a snapshot was used to create the guest
# on the dest host.
self.assertEqual(image_uuid, server['image']['id'])
self.assertEqual(
1, len(self.created_images),
"Unexpected number of images created for image-backed resize")
# Make sure the temporary snapshot image was deleted; we use the
# compute images proxy API here which is deprecated so we force the
# microversion to 2.1.
with utils.temporary_mutation(self.api, microversion='2.1'):
self.api.api_get('/images/%s' % self.created_images[0],
check_response_status=[404])
return server, source_rp_uuid, target_rp_uuid, old_flavor, new_flavor
def _get_instance(self, req, context, server_id):
with utils.temporary_mutation(context, read_deleted='yes'):
return common.get_instance(self.compute_api, context, server_id)
def _get_instance(self, req, context, server_id):
with utils.temporary_mutation(context, read_deleted='yes'):
return common.get_instance(self.compute_api, context, server_id)
def _get_provider_uuid_by_host(self, host):
# We have to temporarily mutate to 2.53 to get the hypervisor UUID.
with utils.temporary_mutation(self.admin_api, microversion='2.53'):
return super(ComputeStatusFilterTest211,
self)._get_provider_uuid_by_host(host)
def test_instance_list_deleted_service_with_no_uuid(self):
"""This test covers the following scenario:
1. create an instance on a host which we'll simulate to be old
by not having a uuid set
2. migrate the instance to the "newer" host that has a service uuid
3. delete the old service/compute node
4. start a new service with the old hostname (still host1); this will
also create a new compute_nodes table record for that host/node
5. migrate the instance back to the host1 service
6. list instances which will try to online migrate the old service uuid
"""
host1 = self.start_service('compute', host='host1')
# Create an instance which will be on host1 since it's the only host.
server_req = self._build_server(networks='none')
server = self.api.post_server({'server': server_req})
self._wait_for_state_change(server, 'ACTIVE')
# Now we start a 2nd compute which is "upgraded" (has a uuid) and
# we'll migrate the instance to that host.
host2 = self.start_service('compute', host='host2')
self.assertIsNotNone(host2.service_ref.uuid)
server = self._migrate_server(server, 'host2')
# Delete the host1 service (which implicitly deletes the host1 compute
# node record).
host1.stop()
self.admin_api.api_delete('/os-services/%s' % host1.service_ref.uuid)
# We should now only have 1 compute service (host2).
compute_services = self.admin_api.api_get(
'/os-services?binary=nova-compute').body['services']
self.assertEqual(1, len(compute_services))
# Make sure the compute node is also gone.
self.admin_api.api_get(
'/os-hypervisors?hypervisor_hostname_pattern=host1',
check_response_status=[404])
# Now recreate the host1 service and compute node by restarting the
# service.
self.restart_compute_service(host1)
# At this point, host1's service should have a uuid.
self.assertIsNotNone(host1.service_ref.uuid)
# Sanity check that there are 3 services in the database, but only 1
# is deleted.
ctxt = nova_context.get_admin_context()
with utils.temporary_mutation(ctxt, read_deleted='yes'):
services = db.service_get_all_by_binary(ctxt, 'nova-compute')
self.assertEqual(3, len(services))
deleted_services = [svc for svc in services if svc['deleted']]
self.assertEqual(1, len(deleted_services))
deleted_service = deleted_services[0]
self.assertEqual('host1', deleted_service['host'])
# Now migrate the instance back to host1.
self._migrate_server(server, 'host1')
# Now null out the service uuid to simulate that the deleted host1
# is old. We have to do this through the DB API directly since the
# Service object won't allow a null uuid field. We also have to do
# this *after* deleting the service via the REST API and migrating the
# server because otherwise that will set a uuid when looking up the
# service.
with utils.temporary_mutation(ctxt, read_deleted='yes'):
service_ref = db.service_update(ctxt, deleted_service['id'],
{'uuid': None})
self.assertIsNone(service_ref['uuid'])
# Finally, list servers as an admin so it joins on services to get host
# information.
servers = self.admin_api.get_servers(detail=True)
for server in servers:
self.assertEqual('UP', server['host_status'])
