def test_obj_make_compatible(self):
cell_mapping_obj = cell_mapping.CellMapping(context=self.context)
fake_cell_mapping_obj = cell_mapping.CellMapping(context=self.context,
uuid=uuids.cell,
disabled=False)
obj_primitive = fake_cell_mapping_obj.obj_to_primitive('1.0')
obj = cell_mapping_obj.obj_from_primitive(obj_primitive)
self.assertIn('uuid', obj)
self.assertEqual(uuids.cell, obj.uuid)
self.assertNotIn('disabled', obj)
def test_get_not_deleted_by_cell_and_project_None(self):
cm = cell_mapping.CellMapping(context=self.context,
uuid=uuidsentinel.cell,
database_connection='fake:///',
transport_url='fake://')
cm.create()
im1 = instance_mapping.InstanceMapping(context=self.context,
project_id='fake-project-1',
cell_mapping=cm,
instance_uuid=uuidsentinel.uid1,
queued_for_delete=False)
im1.create()
im2 = instance_mapping.InstanceMapping(context=self.context,
project_id='fake-project-2',
cell_mapping=cm,
instance_uuid=uuidsentinel.uid2,
queued_for_delete=None)
im2.create()
# testing if it accepts None project_id in the query and
# catches None queued for delete records.
ims = (instance_mapping.InstanceMappingList.
get_not_deleted_by_cell_and_project(self.context,
cm.uuid,
None))
self.assertEqual(2, len(ims))
def test_populate_user_id_instance_get_fail(self, mock_inst_get):
cells = []
celldbs = fixtures.CellDatabases()
# Create two cell databases and map them
for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
cm = cell_mapping.CellMapping(context=self.context, uuid=uuid,
database_connection=uuid,
transport_url='fake://')
cm.create()
cells.append(cm)
celldbs.add_cell_database(uuid)
self.useFixture(celldbs)
# Create one instance per cell
for cell in cells:
with context.target_cell(self.context, cell) as cctxt:
inst = instance.Instance(
cctxt,
project_id=self.context.project_id,
user_id=self.context.user_id)
inst.create()
create_mapping(project_id=self.context.project_id,
user_id=None, cell_id=cell.id,
instance_uuid=inst.uuid)
# Simulate the first cell is down/has some error
mock_inst_get.side_effect = [test.TestingException(),
instance.InstanceList(objects=[inst])]
found, done = instance_mapping.populate_user_id(self.context, 1000)
# Verify we continue to the next cell when a down/error cell is
# encountered.
self.assertEqual(2, found)
self.assertEqual(1, done)
def test_get_not_deleted_by_cell_and_project_limit(self):
cm = cell_mapping.CellMapping(context=self.context,
uuid=uuidsentinel.cell,
database_connection='fake:///',
transport_url='fake://')
cm.create()
pid = self.context.project_id
for uuid in (uuidsentinel.uuid2, uuidsentinel.inst2):
im = instance_mapping.InstanceMapping(context=self.context,
project_id=pid,
cell_mapping=cm,
instance_uuid=uuid,
queued_for_delete=False)
im.create()
ims = (instance_mapping.InstanceMappingList.
get_not_deleted_by_cell_and_project(self.context, cm.uuid, pid))
self.assertEqual(2, len(ims))
ims = (instance_mapping.InstanceMappingList.
get_not_deleted_by_cell_and_project(self.context,
cm.uuid,
pid,
limit=10))
self.assertEqual(2, len(ims))
ims = (instance_mapping.InstanceMappingList.
get_not_deleted_by_cell_and_project(self.context,
cm.uuid,
pid,
limit=1))
self.assertEqual(1, len(ims))
def test_obj_make_compatible(self):
cell_mapping_obj = cell_mapping.CellMapping(context=self.context)
fake_cell_mapping_copy = dict(get_db_mapping())
self.assertIn('disabled', fake_cell_mapping_copy)
cell_mapping_obj.obj_make_compatible(fake_cell_mapping_copy, '1.0')
self.assertIn('uuid', fake_cell_mapping_copy)
self.assertNotIn('disabled', fake_cell_mapping_copy)
def _get_cell_mapping(self):
with db_api.api_context_manager.reader.using(self._context) as session:
cell_map = (session.query(api_models.CellMapping).join(
api_models.HostMapping).filter(
api_models.HostMapping.host == self.host).first())
if cell_map is not None:
return cell_mapping.CellMapping._from_db_object(
self._context, cell_mapping.CellMapping(), cell_map)
def test_get_not_deleted_by_cell_and_project(self):
cells = []
# Create two cells
for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
cm = cell_mapping.CellMapping(context=self.context, uuid=uuid,
database_connection="fake:///",
transport_url='fake://')
cm.create()
cells.append(cm)
uuids = {cells[0]: [uuidsentinel.c1i1, uuidsentinel.c1i2],
cells[1]: [uuidsentinel.c2i1, uuidsentinel.c2i2]}
project_ids = ['fake-project-1', 'fake-project-2']
# Create five instance_mappings such that:
for cell, uuid in uuids.items():
# Both the cells contain a mapping belonging to fake-project-1
im1 = instance_mapping.InstanceMapping(context=self.context,
project_id=project_ids[0], cell_mapping=cell,
instance_uuid=uuid[0], queued_for_delete=False)
im1.create()
# Both the cells contain a mapping belonging to fake-project-2
im2 = instance_mapping.InstanceMapping(context=self.context,
project_id=project_ids[1], cell_mapping=cell,
instance_uuid=uuid[1], queued_for_delete=False)
im2.create()
# The second cell has a third mapping that is queued for deletion
# which belongs to fake-project-1.
if cell.uuid == uuidsentinel.cell2:
im3 = instance_mapping.InstanceMapping(context=self.context,
project_id=project_ids[0], cell_mapping=cell,
instance_uuid=uuidsentinel.qfd, queued_for_delete=True)
im3.create()
# Get not queued for deletion mappings from cell1 belonging to
# fake-project-2.
ims = (instance_mapping.InstanceMappingList.
get_not_deleted_by_cell_and_project(
self.context, cells[0].uuid, 'fake-project-2'))
# This will give us one mapping from cell1
self.assertEqual([uuidsentinel.c1i2],
sorted([m.instance_uuid for m in ims]))
self.assertIn('cell_mapping', ims[0])
# Get not queued for deletion mappings from cell2 belonging to
# fake-project-1.
ims = (instance_mapping.InstanceMappingList.
get_not_deleted_by_cell_and_project(
self.context, cells[1].uuid, 'fake-project-1'))
# This will give us one mapping from cell2. Note that even if
# there are two mappings belonging to fake-project-1 inside cell2,
# only the one not queued for deletion is returned.
self.assertEqual([uuidsentinel.c2i1],
sorted([m.instance_uuid for m in ims]))
# Try getting a mapping belonging to a non-existing project_id.
ims = (instance_mapping.InstanceMappingList.
get_not_deleted_by_cell_and_project(
self.context, cells[0].uuid, 'fake-project-3'))
# Since no mappings belong to fake-project-3, nothing is returned.
self.assertEqual([], sorted([m.instance_uuid for m in ims]))
def _get_cell_mapping(self):
session = db_api.get_api_session()
with session.begin():
cell_map = (session.query(api_models.CellMapping).join(
api_models.HostMapping).filter(
api_models.HostMapping.host == self.host).first())
if cell_map is not None:
return cell_mapping.CellMapping._from_db_object(
self._context, cell_mapping.CellMapping(), cell_map)
def _from_db_object(context, instance_mapping, db_instance_mapping):
for key in instance_mapping.fields:
db_value = db_instance_mapping.get(key)
if key == 'cell_mapping':
# cell_mapping can be None indicating that the instance has
# not been scheduled yet.
if db_value:
db_value = cell_mapping.CellMapping._from_db_object(
context, cell_mapping.CellMapping(), db_value)
setattr(instance_mapping, key, db_value)
instance_mapping.obj_reset_changes()
instance_mapping._context = context
return instance_mapping
def _from_db_object(context, host_mapping, db_host_mapping):
for key in host_mapping.fields:
db_value = db_host_mapping.get(key)
if key == "cell_mapping":
# NOTE(dheeraj): If cell_mapping is stashed in db object
# we load it here. Otherwise, lazy loading will happen
# when .cell_mapping is accessd later
if not db_value:
continue
db_value = cell_mapping.CellMapping._from_db_object(
host_mapping._context, cell_mapping.CellMapping(),
db_value)
setattr(host_mapping, key, db_value)
host_mapping.obj_reset_changes()
host_mapping._context = context
return host_mapping
def _from_db_object(context, instance_mapping, db_instance_mapping):
for key in instance_mapping.fields:
db_value = db_instance_mapping.get(key)
if key == 'cell_mapping':
# cell_mapping can be None indicating that the instance has
# not been scheduled yet.
if db_value:
db_value = cell_mapping.CellMapping._from_db_object(
context, cell_mapping.CellMapping(), db_value)
if key == 'user_id' and db_value is None:
# NOTE(melwitt): If user_id is NULL, we can't set the field
# because it's non-nullable. We don't plan for any code to read
# the user_id field at this time, so skip setting it.
continue
setattr(instance_mapping, key, db_value)
instance_mapping.obj_reset_changes()
instance_mapping._context = context
return instance_mapping
def test_get_by_cell_and_project(self):
cells = []
# Create two cells
for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
cm = cell_mapping.CellMapping(context=self.context,
uuid=uuid,
database_connection="fake:///",
transport_url='fake://')
cm.create()
cells.append(cm)
# With each cell having two instance_mappings of two project_ids.
uuids = {
cells[0].id: [uuidsentinel.c1i1, uuidsentinel.c1i2],
cells[1].id: [uuidsentinel.c2i1, uuidsentinel.c2i2]
}
project_ids = ['fake-project-1', 'fake-project-2']
for cell_id, uuid in uuids.items():
instance_mapping.InstanceMapping._create_in_db(
self.context, {
'project_id': project_ids[0],
'cell_id': cell_id,
'instance_uuid': uuid[0]
})
instance_mapping.InstanceMapping._create_in_db(
self.context, {
'project_id': project_ids[1],
'cell_id': cell_id,
'instance_uuid': uuid[1]
})
ims = instance_mapping.InstanceMappingList.get_by_cell_and_project(
self.context, cells[0].id, 'fake-project-2')
self.assertEqual([uuidsentinel.c1i2],
sorted([m.instance_uuid for m in ims]))
ims = instance_mapping.InstanceMappingList.get_by_cell_and_project(
self.context, cells[1].id, 'fake-project-1')
self.assertEqual([uuidsentinel.c2i1],
sorted([m.instance_uuid for m in ims]))
ims = instance_mapping.InstanceMappingList.get_by_cell_and_project(
self.context, cells[0].id, 'fake-project-3')
self.assertEqual([], sorted([m.instance_uuid for m in ims]))
def test_modify_cell_mapping(self):
inst_mapping = instance_mapping.InstanceMapping(context=self.context)
inst_mapping.instance_uuid = uuidutils.generate_uuid()
inst_mapping.project_id = self.context.project_id
inst_mapping.cell_mapping = None
inst_mapping.create()
c_mapping = cell_mapping.CellMapping(self.context,
uuid=uuidutils.generate_uuid(),
name="cell0",
transport_url="none:///",
database_connection="fake:///")
c_mapping.create()
inst_mapping.cell_mapping = c_mapping
inst_mapping.save()
result_mapping = instance_mapping.InstanceMapping.get_by_instance_uuid(
self.context, inst_mapping.instance_uuid)
self.assertEqual(result_mapping.cell_mapping.id, c_mapping.id)
def setUp(self):
super(CellMappingTestCase, self).setUp()
self.useFixture(fixtures.Database(database='api'))
self.context = context.RequestContext('fake-user', 'fake-project')
self.mapping_obj = cell_mapping.CellMapping()
self.uuid = uuidutils.generate_uuid()
def setUp(self):
super(HostMappingTestCase, self).setUp()
self.useFixture(fixtures.Database(database='api'))
self.context = context.RequestContext('fake-user', 'fake-project')
self.mapping_obj = host_mapping.HostMapping()
self.cell_mapping_obj = cell_mapping.CellMapping()
def test_populate_user_id(self, mock_log_warning):
cells = []
celldbs = fixtures.CellDatabases()
# Create two cell databases and map them
for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
cm = cell_mapping.CellMapping(context=self.context,
uuid=uuid,
database_connection=uuid,
transport_url='fake://')
cm.create()
cells.append(cm)
celldbs.add_cell_database(uuid)
self.useFixture(celldbs)
# Create 5 instances per cell
for cell in cells:
for i in range(0, 5):
with context.target_cell(self.context, cell) as cctxt:
inst = instance.Instance(
cctxt,
project_id=self.context.project_id,
user_id=self.context.user_id)
inst.create()
# Make every other mapping have a NULL user_id
# Will be a total of four mappings with NULL user_id
user_id = self.context.user_id if i % 2 == 0 else None
create_mapping(project_id=self.context.project_id,
user_id=user_id,
cell_id=cell.id,
instance_uuid=inst.uuid)
# Create a SOFT_DELETED instance with a user_id=None instance mapping.
# This should get migrated.
with context.target_cell(self.context, cells[0]) as cctxt:
inst = instance.Instance(cctxt,
project_id=self.context.project_id,
user_id=self.context.user_id,
vm_state=vm_states.SOFT_DELETED)
inst.create()
create_mapping(project_id=self.context.project_id,
user_id=None,
cell_id=cells[0].id,
instance_uuid=inst.uuid,
queued_for_delete=True)
# Create a deleted instance with a user_id=None instance mapping.
# This should get migrated.
with context.target_cell(self.context, cells[1]) as cctxt:
inst = instance.Instance(cctxt,
project_id=self.context.project_id,
user_id=self.context.user_id)
inst.create()
inst.destroy()
create_mapping(project_id=self.context.project_id,
user_id=None,
cell_id=cells[1].id,
instance_uuid=inst.uuid,
queued_for_delete=True)
# Create an instance mapping for an instance not yet scheduled. It
# should not get migrated because we won't know what user_id to use.
unscheduled = create_mapping(project_id=self.context.project_id,
user_id=None,
cell_id=None)
# Create two instance mappings for instances that no longer exist.
# Example: residue from a manual cleanup or after a periodic compute
# purge and before a database archive. This record should not get
# migrated.
nonexistent = []
for i in range(2):
nonexistent.append(
create_mapping(project_id=self.context.project_id,
user_id=None,
cell_id=cells[i].id,
instance_uuid=uuidutils.generate_uuid()))
# Create an instance mapping simulating a virtual interface migration
# marker instance which has had map_instances run on it.
# This should not be found by the migration.
create_mapping(project_id=virtual_interface.FAKE_UUID, user_id=None)
found, done = instance_mapping.populate_user_id(self.context, 2)
# Two needed fixing, and honored the limit.
self.assertEqual(2, found)
self.assertEqual(2, done)
found, done = instance_mapping.populate_user_id(self.context, 1000)
# Only four left were fixable. The fifth instance found has no
# cell and cannot be migrated yet. The 6th and 7th instances found have
# no corresponding instance records and cannot be migrated.
self.assertEqual(7, found)
self.assertEqual(4, done)
# Verify the orphaned instance mappings warning log message was only
# emitted once.
mock_log_warning.assert_called_once()
# Check that we have only the expected number of records with
# user_id set. We created 10 instances (5 per cell with 2 per cell
# with NULL user_id), 1 SOFT_DELETED instance with NULL user_id,
# 1 deleted instance with NULL user_id, and 1 not-yet-scheduled
# instance with NULL user_id.
# We expect 12 of them to have user_id set after migration (15 total,
# with the not-yet-scheduled instance and the orphaned instance
# mappings ignored).
ims = instance_mapping.InstanceMappingList.get_by_project_id(
self.context, self.context.project_id)
self.assertEqual(12, len([im for im in ims if 'user_id' in im]))
# Check that one instance mapping record (not yet scheduled) has not
# been migrated by this script.
# Check that two other instance mapping records (no longer existing
# instances) have not been migrated by this script.
self.assertEqual(15, len(ims))
# Set the cell and create the instance for the mapping without a cell,
# then run the migration again.
unscheduled = instance_mapping.InstanceMapping.get_by_instance_uuid(
self.context, unscheduled['instance_uuid'])
unscheduled.cell_mapping = cells[0]
unscheduled.save()
with context.target_cell(self.context, cells[0]) as cctxt:
inst = instance.Instance(cctxt,
uuid=unscheduled.instance_uuid,
project_id=self.context.project_id,
user_id=self.context.user_id)
inst.create()
found, done = instance_mapping.populate_user_id(self.context, 1000)
# Should have found the not-yet-scheduled instance and the orphaned
# instance mappings.
self.assertEqual(3, found)
# Should have only migrated the not-yet-schedule instance.
self.assertEqual(1, done)
# Delete the orphaned instance mapping (simulate manual cleanup by an
# operator).
for db_im in nonexistent:
nonexist = instance_mapping.InstanceMapping.get_by_instance_uuid(
self.context, db_im['instance_uuid'])
nonexist.destroy()
# Run the script one last time to make sure it finds nothing left to
# migrate.
found, done = instance_mapping.populate_user_id(self.context, 1000)
self.assertEqual(0, found)
self.assertEqual(0, done)
def test_populate_queued_for_delete(self):
cells = []
celldbs = fixtures.CellDatabases()
# Create two cell databases and map them
for uuid in (uuidsentinel.cell1, uuidsentinel.cell2):
cm = cell_mapping.CellMapping(context=self.context,
uuid=uuid,
database_connection=uuid,
transport_url='fake://')
cm.create()
cells.append(cm)
celldbs.add_cell_database(uuid)
self.useFixture(celldbs)
# Create 5 instances per cell, two deleted, one with matching
# queued_for_delete in the instance mapping
for cell in cells:
for i in range(0, 5):
# Instance 4 should be SOFT_DELETED
vm_state = (vm_states.SOFT_DELETED
if i == 4 else vm_states.ACTIVE)
# Instance 2 should already be marked as queued_for_delete
qfd = True if i == 2 else None
with context.target_cell(self.context, cell) as cctxt:
inst = instance.Instance(
cctxt,
vm_state=vm_state,
project_id=self.context.project_id,
user_id=self.context.user_id)
inst.create()
if i in (2, 3):
# Instances 2 and 3 are hard-deleted
inst.destroy()
instance_mapping.InstanceMapping._create_in_db(
self.context, {
'project_id': self.context.project_id,
'cell_id': cell.id,
'queued_for_delete': qfd,
'instance_uuid': inst.uuid
})
done, total = instance_mapping.populate_queued_for_delete(
self.context, 2)
# First two needed fixing, and honored the limit
self.assertEqual(2, done)
self.assertEqual(2, total)
done, total = instance_mapping.populate_queued_for_delete(
self.context, 1000)
# Last six included two that were already done, and spanned to the
# next cell
self.assertEqual(6, done)
self.assertEqual(6, total)
mappings = instance_mapping.InstanceMappingList.get_by_project_id(
self.context, self.context.project_id)
# Check that we have only the expected number of records with
# True/False (which implies no NULL records).
# Six deleted instances
self.assertEqual(
6, len([im for im in mappings if im.queued_for_delete is True]))
# Four non-deleted instances
self.assertEqual(
4, len([im for im in mappings if im.queued_for_delete is False]))
# Run it again to make sure we don't query the cell database for
# instances if we didn't get any un-migrated mappings.
with mock.patch('nova.objects.InstanceList.get_by_filters',
new_callable=mock.NonCallableMock):
done, total = instance_mapping.populate_queued_for_delete(
self.context, 1000)
self.assertEqual(0, done)
self.assertEqual(0, total)