def test_delete_fast_if_host_not_set(self):
inst = self._create_instance_obj()
inst.host = ''
db_inst = obj_base.obj_to_primitive(inst)
updates = {'progress': 0, 'task_state': task_states.DELETING}
new_inst = dict(db_inst, **updates)
self.mox.StubOutWithMock(db,
'block_device_mapping_get_all_by_instance')
self.mox.StubOutWithMock(db,
'instance_update_and_get_original')
self.mox.StubOutWithMock(db, 'constraint')
self.mox.StubOutWithMock(db, 'instance_destroy')
self.mox.StubOutWithMock(self.compute_api, '_create_reservations')
db.block_device_mapping_get_all_by_instance(self.context,
inst.uuid).AndReturn([])
db.instance_update_and_get_original(
self.context, inst.uuid, updates).AndReturn((db_inst, new_inst))
self.compute_api._create_reservations(self.context,
db_inst, new_inst,
inst.project_id,
inst.user_id).AndReturn(None)
db.constraint(host=mox.IgnoreArg()).AndReturn('constraint')
db.instance_destroy(self.context, inst.uuid, 'constraint')
if self.is_cells:
self.mox.StubOutWithMock(self.compute_api, '_cast_to_cells')
self.compute_api._cast_to_cells(
self.context, db_inst, 'delete')
self.mox.ReplayAll()
self.compute_api.delete(self.context, db_inst)
def test_prep_resize_exception_host_in_error_state_and_raise(self):
fake_instance_uuid = "fake-instance-id"
self._mox_schedule_method_helper("schedule_prep_resize")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
request_spec = {"instance_properties": {"uuid": fake_instance_uuid}}
kwargs = {
"context": self.context,
"image": "fake_image",
"request_spec": request_spec,
"filter_properties": "fake_props",
"instance": "fake_instance",
"instance_type": "fake_type",
"reservations": list("fake_res"),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(test.TestingException("something happened"))
inst = {"vm_state": "", "task_state": ""}
db.instance_update_and_get_original(
self.context, fake_instance_uuid, {"vm_state": vm_states.ERROR, "task_state": None}
).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, fake_instance_uuid, mox.IsA(test.TestingException), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.assertRaises(test.TestingException, self.manager.prep_resize, **kwargs)
def test_live_migration_set_vmstate_error(self):
inst = {"uuid": "fake-instance-id", "vm_state": vm_states.ACTIVE}
dest = "fake_host"
block_migration = False
disk_over_commit = False
self._mox_schedule_method_helper("schedule_live_migration")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.manager.driver.schedule_live_migration(
self.context, inst, dest, block_migration, disk_over_commit
).AndRaise(ValueError)
db.instance_update_and_get_original(self.context, inst["uuid"], {"vm_state": vm_states.ERROR}).AndReturn(
(inst, inst)
)
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), inst, mox.IsA(ValueError), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.stub_out_client_exceptions()
self.assertRaises(
ValueError, self.manager.live_migration, self.context, inst, dest, block_migration, disk_over_commit
)
def test_live_migration_set_vmstate_error(self):
inst = {"uuid": "fake-instance-id",
"vm_state": vm_states.ACTIVE, }
dest = 'fake_host'
block_migration = False
disk_over_commit = False
pclm = "pclm"
self.mox.StubOutWithMock(self.manager, '_schedule_live_migration')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.manager._schedule_live_migration(self.context,
inst, dest, block_migration, disk_over_commit, pclm).AndRaise(
ValueError)
db.instance_update_and_get_original(self.context, inst["uuid"],
{"vm_state": vm_states.ERROR,
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
mox.IsA(conductor_api.LocalAPI), inst,
mox.IsA(ValueError),
mox.IgnoreArg())
self.mox.ReplayAll()
self.manager = utils.ExceptionHelper(self.manager)
self.assertRaises(ValueError,
self.manager.live_migration,
self.context, inst, dest, block_migration,
disk_over_commit, pclm)
def test_run_instance_no_hosts(self):
def _fake_empty_call_zone_method(*args, **kwargs):
return []
sched = fakes.FakeFilterScheduler()
uuid = 'fake-uuid1'
fake_context = context.RequestContext('user', 'project')
instance_properties = {'project_id': 1, 'os_type': 'Linux'}
request_spec = {'instance_type': {'memory_mb': 1, 'root_gb': 1,
'ephemeral_gb': 0},
'instance_properties': instance_properties,
'instance_uuids': [uuid]}
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
compute_utils.add_instance_fault_from_exc(fake_context,
uuid, mox.IsA(exception.NoValidHost), mox.IgnoreArg())
db.instance_update_and_get_original(fake_context, uuid,
{'vm_state': vm_states.ERROR,
'task_state': None}).AndReturn(({}, {}))
self.mox.ReplayAll()
sched.schedule_run_instance(
fake_context, request_spec, None, None, None, None, {})
def test_run_instance_non_admin(self):
self.was_admin = False
def fake_get(context, *args, **kwargs):
# make sure this is called with admin context, even though
# we're using user context below
self.was_admin = context.is_admin
return {}
sched = fakes.FakeFilterScheduler()
self.stubs.Set(sched.host_manager, 'get_all_host_states', fake_get)
fake_context = context.RequestContext('user', 'project')
uuid = 'fake-uuid1'
instance_properties = {'project_id': 1, 'os_type': 'Linux'}
request_spec = {'instance_type': {'memory_mb': 1, 'local_gb': 1},
'instance_properties': instance_properties,
'instance_uuids': [uuid]}
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
compute_utils.add_instance_fault_from_exc(fake_context,
uuid, mox.IsA(exception.NoValidHost), mox.IgnoreArg())
db.instance_update_and_get_original(fake_context, uuid,
{'vm_state': vm_states.ERROR,
'task_state': None}).AndReturn(({}, {}))
self.mox.ReplayAll()
sched.schedule_run_instance(
fake_context, request_spec, None, None, None, None, {})
self.assertTrue(self.was_admin)
def test_prep_resize_exception_host_in_error_state_and_raise(self):
"""Test that a NoValidHost exception for prep_resize puts
the instance in ACTIVE state
"""
fake_instance_uuid = 'fake-instance-id'
self._mox_schedule_method_helper('schedule_prep_resize')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_properties':
{'uuid': fake_instance_uuid}}
self.fake_kwargs['request_spec'] = request_spec
self.manager.driver.schedule_prep_resize(self.context,
*self.fake_args, **self.fake_kwargs).AndRaise(
self.AnException('something happened'))
inst = {
"vm_state": "",
"task_state": "",
}
db.instance_update_and_get_original(self.context, fake_instance_uuid,
{"vm_state": vm_states.ERROR}).AndReturn((inst, inst))
self.mox.ReplayAll()
self.assertRaises(self.AnException, self.manager.prep_resize,
self.context, self.topic,
*self.fake_args, **self.fake_kwargs)
def test_unshelve_volume_backed(self):
db_instance = jsonutils.to_primitive(self._create_fake_instance())
host = 'fake-mini'
cur_time = timeutils.utcnow()
cur_time_tz = cur_time.replace(tzinfo=iso8601.iso8601.Utc())
timeutils.set_time_override(cur_time)
self.compute.run_instance(self.context, instance=db_instance)
instance = instance_obj.Instance.get_by_uuid(
self.context, db_instance['uuid'],
expected_attrs=['metadata', 'system_metadata'])
instance.task_state = task_states.UNSHELVING
instance.save()
sys_meta = dict(instance.system_metadata)
sys_meta['shelved_at'] = timeutils.strtime(at=cur_time)
sys_meta['shelved_image_id'] = None
sys_meta['shelved_host'] = host
self.mox.StubOutWithMock(self.compute, '_notify_about_instance_usage')
self.mox.StubOutWithMock(self.compute, '_prep_block_device')
self.mox.StubOutWithMock(self.compute.driver, 'spawn')
self.mox.StubOutWithMock(self.compute, '_get_power_state')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.compute._notify_about_instance_usage(self.context, instance,
'unshelve.start')
db.instance_update_and_get_original(self.context, instance['uuid'],
{'task_state': task_states.SPAWNING},
update_cells=False,
columns_to_join=['metadata', 'system_metadata']
).AndReturn((db_instance, db_instance))
self.compute._prep_block_device(self.context, instance,
[]).AndReturn('fake_bdm')
db_instance['key_data'] = None
db_instance['auto_disk_config'] = None
self.compute.driver.spawn(self.context, instance, None,
injected_files=[], admin_password=None,
network_info=[],
block_device_info='fake_bdm')
self.compute._get_power_state(self.context, instance).AndReturn(123)
db.instance_update_and_get_original(self.context, instance['uuid'],
{'power_state': 123,
'vm_state': vm_states.ACTIVE,
'task_state': None,
'key_data': None,
'auto_disk_config': False,
'expected_task_state': task_states.SPAWNING,
'launched_at': cur_time_tz},
update_cells=False,
columns_to_join=['metadata', 'system_metadata']
).AndReturn((db_instance, db_instance))
self.compute._notify_about_instance_usage(self.context, instance,
'unshelve.end')
self.mox.ReplayAll()
self.compute.unshelve_instance(self.context, instance, image=None)
self.mox.VerifyAll()
self.mox.UnsetStubs()
self.compute.terminate_instance(self.context, instance=instance)
def _wait_for_boot():
try:
LOG.debug(_("Key is injected but instance is not running yet"), instance=instance)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance["id"], {"vm_state": vm_states.BUILDING}
)
notifications.send_update(context, old_ref, new_ref)
state = self._conn.create_domain(xml_dict, bpath)
if state == power_state.RUNNING:
LOG.debug(_("instance %s: booted"), instance["name"], instance=instance)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance["id"], {"vm_state": vm_states.ACTIVE}
)
notifications.send_update(context, old_ref, new_ref)
LOG.debug(_("~~~~~~ current state = %s ~~~~~~"), state, instance=instance)
LOG.debug(_("instance %s spawned successfully"), instance["name"], instance=instance)
else:
LOG.debug(_("instance %s:not booted"), instance["name"], instance=instance)
except Exception:
LOG.exception(_("Baremetal assignment is overcommitted."), instance=instance)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance["id"], {"vm_state": vm_states.ERROR, "power_state": power_state.FAILED}
)
notifications.send_update(context, old_ref, new_ref)
timer.stop()
def test_prep_resize_no_valid_host_back_in_active_state(self):
"""Test that a NoValidHost exception for prep_resize puts
the instance in ACTIVE state
"""
fake_instance_uuid = 'fake-instance-id'
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper('schedule_prep_resize')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_type': 'fake_type',
'instance_uuids': [fake_instance_uuid],
'instance_properties': {'uuid': fake_instance_uuid}}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': 'fake_instance',
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(
exception.NoValidHost(reason=""))
db.instance_update_and_get_original(self.context, fake_instance_uuid,
{"vm_state": vm_states.ACTIVE, "task_state": None}).AndReturn(
(inst, inst))
self.mox.ReplayAll()
# FIXME(comstud): Remove 'update_db' on future RPC version bump.
self.manager.prep_resize(update_db=False, **kwargs)
def test_live_migration_basic(self):
"""Test basic schedule_live_migration functionality"""
self.mox.StubOutWithMock(db, 'instance_get')
self.mox.StubOutWithMock(self.driver, '_live_migration_src_check')
self.mox.StubOutWithMock(self.driver, '_live_migration_dest_check')
self.mox.StubOutWithMock(self.driver, '_live_migration_common_check')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(driver, 'cast_to_compute_host')
dest = 'fake_host2'
block_migration = False
disk_over_commit = False
instance = self._live_migration_instance()
db.instance_get(self.context, instance['id']).AndReturn(instance)
self.driver._live_migration_src_check(self.context, instance)
self.driver._live_migration_dest_check(self.context, instance,
dest, block_migration, disk_over_commit)
self.driver._live_migration_common_check(self.context, instance,
dest, block_migration, disk_over_commit)
db.instance_update_and_get_original(self.context, instance['id'],
{"task_state": task_states.MIGRATING}).AndReturn(
(instance, instance))
driver.cast_to_compute_host(self.context, instance['host'],
'live_migration', update_db=False,
instance_id=instance['id'], dest=dest,
block_migration=block_migration)
self.mox.ReplayAll()
self.driver.schedule_live_migration(self.context,
instance_id=instance['id'], dest=dest,
block_migration=block_migration,
disk_over_commit=disk_over_commit)
def test_save_rename_sends_notification(self):
# Tests that simply changing the 'display_name' on the instance
# will send a notification.
self.flags(enable=False, group='cells')
old_ref = dict(self.fake_instance, display_name='hello')
fake_uuid = old_ref['uuid']
expected_updates = dict(display_name='goodbye')
new_ref = dict(old_ref, **expected_updates)
self.mox.StubOutWithMock(db, 'instance_get_by_uuid')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(notifications, 'send_update')
db.instance_get_by_uuid(self.context, fake_uuid,
columns_to_join=['info_cache',
'security_groups'],
use_slave=False
).AndReturn(old_ref)
db.instance_update_and_get_original(
self.context, fake_uuid, expected_updates, update_cells=False,
columns_to_join=['info_cache', 'security_groups',
'system_metadata']
).AndReturn((old_ref, new_ref))
notifications.send_update(self.context, mox.IgnoreArg(),
mox.IgnoreArg())
self.mox.ReplayAll()
inst = instance.Instance.get_by_uuid(self.context, old_ref['uuid'],
use_slave=False)
self.assertEqual('hello', inst.display_name)
inst.display_name = 'goodbye'
inst.save()
self.assertEqual('goodbye', inst.display_name)
self.assertEqual(set([]), inst.obj_what_changed())
def test_prep_resize_no_valid_host_back_in_active_state(self):
fake_instance_uuid = "fake-instance-id"
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper("schedule_prep_resize")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
request_spec = {
"instance_type": "fake_type",
"instance_uuids": [fake_instance_uuid],
"instance_properties": {"uuid": fake_instance_uuid},
}
kwargs = {
"context": self.context,
"image": "fake_image",
"request_spec": request_spec,
"filter_properties": "fake_props",
"instance": "fake_instance",
"instance_type": "fake_type",
"reservations": list("fake_res"),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(exception.NoValidHost(reason=""))
db.instance_update_and_get_original(
self.context, fake_instance_uuid, {"vm_state": vm_states.ACTIVE, "task_state": None}
).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, fake_instance_uuid, mox.IsA(exception.NoValidHost), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.manager.prep_resize(**kwargs)
def test_prep_resize_no_valid_host_back_in_active_state(self):
fake_instance_uuid = 'fake-instance-id'
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper('schedule_prep_resize')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_type': 'fake_type',
'instance_uuids': [fake_instance_uuid],
'instance_properties': {'uuid': fake_instance_uuid}}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': 'fake_instance',
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(
exception.NoValidHost(reason=""))
db.instance_update_and_get_original(self.context, fake_instance_uuid,
{"vm_state": vm_states.ACTIVE, "task_state": None}).AndReturn(
(inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
fake_instance_uuid, mox.IsA(exception.NoValidHost),
mox.IgnoreArg())
self.mox.ReplayAll()
self.manager.prep_resize(**kwargs)
def test_basic_schedule_run_instance_no_hosts(self):
ctxt = context.RequestContext('fake', 'fake', False)
ctxt_elevated = 'fake-context-elevated'
fake_args = (1, 2, 3)
uuid = 'fake-uuid1'
instance_opts = {'fake_opt1': 'meow', 'launch_index': -1}
request_spec = {'instance_uuids': [uuid],
'instance_properties': instance_opts}
self.mox.StubOutWithMock(ctxt, 'elevated')
self.mox.StubOutWithMock(self.driver, 'hosts_up')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
# instance 1
ctxt.elevated().AndReturn(ctxt_elevated)
self.driver.hosts_up(ctxt_elevated, 'compute').AndReturn([])
compute_utils.add_instance_fault_from_exc(ctxt,
uuid, mox.IsA(exception.NoValidHost), mox.IgnoreArg())
db.instance_update_and_get_original(ctxt, uuid,
{'vm_state': vm_states.ERROR,
'task_state': None}).AndReturn(({}, {}))
self.mox.ReplayAll()
self.driver.schedule_run_instance(
ctxt, request_spec, None, None, None, None, {})
def test_save_rename_sends_notification(self):
# Tests that simply changing the 'display_name' on the instance
# will send a notification.
self.flags(enable=False, group="cells")
old_ref = dict(self.fake_instance, display_name="hello")
fake_uuid = old_ref["uuid"]
expected_updates = dict(display_name="goodbye")
new_ref = dict(old_ref, **expected_updates)
self.mox.StubOutWithMock(db, "instance_get_by_uuid")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.mox.StubOutWithMock(notifications, "send_update")
db.instance_get_by_uuid(
self.context, fake_uuid, columns_to_join=["info_cache", "security_groups"], use_slave=False
).AndReturn(old_ref)
db.instance_update_and_get_original(
self.context,
fake_uuid,
expected_updates,
update_cells=False,
columns_to_join=["info_cache", "security_groups"],
).AndReturn((old_ref, new_ref))
notifications.send_update(self.context, mox.IgnoreArg(), mox.IgnoreArg())
self.mox.ReplayAll()
inst = instance.Instance.get_by_uuid(self.context, old_ref["uuid"], use_slave=False)
self.assertEqual("hello", inst.display_name)
inst.display_name = "goodbye"
inst.save()
self.assertEqual("goodbye", inst.display_name)
self.assertEqual(set([]), inst.obj_what_changed())
def test_live_migration_compute_service_notavailable(self):
inst = {"uuid": "fake-instance-id",
"vm_state": vm_states.ACTIVE,
"task_state": task_states.MIGRATING, }
dest = 'fake_host'
block_migration = False
disk_over_commit = False
self._mox_schedule_method_helper('schedule_live_migration')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.manager.driver.schedule_live_migration(self.context,
inst, dest, block_migration, disk_over_commit).AndRaise(
exception.ComputeServiceUnavailable(host="src"))
db.instance_update_and_get_original(self.context, inst["uuid"],
{"vm_state": inst['vm_state'],
"task_state": None,
"expected_task_state": task_states.MIGRATING,
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
mox.IsA(conductor_api.LocalAPI), inst,
mox.IsA(exception.ComputeServiceUnavailable),
mox.IgnoreArg())
self.mox.ReplayAll()
self.stub_out_client_exceptions()
self.assertRaises(exception.ComputeServiceUnavailable,
self.manager.live_migration,
self.context, inst, dest, block_migration,
disk_over_commit)
def test_live_migration_compute_service_notavailable(self):
inst = {"uuid": "fake-instance-id", "vm_state": vm_states.ACTIVE, "task_state": task_states.MIGRATING}
dest = "fake_host"
block_migration = False
disk_over_commit = False
self.mox.StubOutWithMock(self.manager, "_schedule_live_migration")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.manager._schedule_live_migration(self.context, inst, dest, block_migration, disk_over_commit).AndRaise(
exception.ComputeServiceUnavailable(host="src")
)
db.instance_update_and_get_original(
self.context,
inst["uuid"],
{"vm_state": inst["vm_state"], "task_state": None, "expected_task_state": task_states.MIGRATING},
).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, inst, mox.IsA(exception.ComputeServiceUnavailable), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.manager = utils.ExceptionHelper(self.manager)
self.assertRaises(
exception.ComputeServiceUnavailable,
self.manager.live_migration,
self.context,
inst,
dest,
block_migration,
disk_over_commit,
)
def test_live_migration_schedule_novalidhost(self):
inst = {"uuid": "fake-instance-id",
"vm_state": vm_states.ACTIVE,
"task_state": task_states.MIGRATING, }
dest = None
block_migration = False
disk_over_commit = False
pclm = "pclm"
self.mox.StubOutWithMock(self.manager, '_schedule_live_migration')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.manager._schedule_live_migration(self.context,
inst, dest, block_migration, disk_over_commit, pclm).AndRaise(
exception.NoValidHost(reason=""))
db.instance_update_and_get_original(self.context, inst["uuid"],
{"vm_state": inst['vm_state'],
"task_state": None,
"expected_task_state": task_states.MIGRATING,
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
mox.IsA(conductor_api.LocalAPI), inst,
mox.IsA(exception.NoValidHost),
mox.IgnoreArg())
self.mox.ReplayAll()
self.manager = utils.ExceptionHelper(self.manager)
self.assertRaises(exception.NoValidHost,
self.manager.live_migration,
self.context, inst, dest, block_migration,
disk_over_commit, pclm)
def _wait_for_boot():
try:
LOG.debug(_("Key is injected but instance is not running yet"),
instance=instance)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance['id'],
{'vm_state': vm_states.BUILDING})
notifications.send_update(context, old_ref, new_ref)
state = self._conn.create_domain(xml_dict, bpath)
if state == power_state.RUNNING:
LOG.debug(_('instance %s: booted'), instance['name'],
instance=instance)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance['id'],
{'vm_state': vm_states.ACTIVE})
notifications.send_update(context, old_ref, new_ref)
LOG.debug(_('~~~~~~ current state = %s ~~~~~~'), state,
instance=instance)
LOG.debug(_("instance %s spawned successfully"),
instance['name'], instance=instance)
else:
LOG.debug(_('instance %s:not booted'), instance['name'],
instance=instance)
except Exception as Exn:
LOG.debug(_("Bremetal assignment is overcommitted."),
instance=instance)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance['id'],
{'vm_state': vm_states.OVERCOMMIT,
'power_state': power_state.SUSPENDED})
notifications.send_update(context, old_ref, new_ref)
timer.stop()
def test_prep_resize_no_valid_host_back_in_active_state(self):
"""Test that a NoValidHost exception for prep_resize puts
the instance in ACTIVE state
"""
fake_instance_uuid = 'fake-instance-id'
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper('schedule_prep_resize')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_properties':
{'uuid': fake_instance_uuid}}
self.fake_kwargs['request_spec'] = request_spec
self.manager.driver.schedule_prep_resize(self.context,
*self.fake_args, **self.fake_kwargs).AndRaise(
exception.NoValidHost(reason=""))
db.instance_update_and_get_original(self.context, fake_instance_uuid,
{"vm_state": vm_states.ACTIVE, "task_state": None}).AndReturn(
(inst, inst))
self.mox.ReplayAll()
self.manager.prep_resize(self.context, self.topic,
*self.fake_args, **self.fake_kwargs)
def test_live_migration_schedule_novalidhost(self):
inst = {"uuid": "fake-instance-id", "vm_state": vm_states.ACTIVE, "task_state": task_states.MIGRATING}
dest = None
block_migration = False
disk_over_commit = False
self._mox_schedule_method_helper("schedule_live_migration")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.manager.driver.schedule_live_migration(
self.context, inst, dest, block_migration, disk_over_commit
).AndRaise(exception.NoValidHost(reason=""))
db.instance_update_and_get_original(
self.context,
inst["uuid"],
{"vm_state": inst["vm_state"], "task_state": None, "expected_task_state": task_states.MIGRATING},
).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), inst, mox.IsA(exception.NoValidHost), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.stub_out_client_exceptions()
self.assertRaises(
exception.NoValidHost,
self.manager.live_migration,
self.context,
inst,
dest,
block_migration,
disk_over_commit,
)
def _save_test_helper(self, cell_type, save_kwargs):
"""Common code for testing save() for cells/non-cells."""
if cell_type:
self.flags(enable=True, cell_type=cell_type, group="cells")
else:
self.flags(enable=False, group="cells")
ctxt = context.get_admin_context()
old_ref = dict(self.fake_instance, host="oldhost", user_data="old", vm_state="old", task_state="old")
fake_uuid = old_ref["uuid"]
expected_updates = dict(vm_state="meow", task_state="wuff", user_data="new")
new_ref = dict(old_ref, host="newhost", **expected_updates)
exp_vm_state = save_kwargs.get("expected_vm_state")
exp_task_state = save_kwargs.get("expected_task_state")
admin_reset = save_kwargs.get("admin_state_reset", False)
if exp_vm_state:
expected_updates["expected_vm_state"] = exp_vm_state
if exp_task_state:
expected_updates["expected_task_state"] = exp_task_state
self.mox.StubOutWithMock(db, "instance_get_by_uuid")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.mox.StubOutWithMock(db, "instance_info_cache_update")
cells_api_mock = self.mox.CreateMock(cells_rpcapi.CellsAPI)
self.mox.StubOutWithMock(cells_api_mock, "instance_update_at_top")
self.mox.StubOutWithMock(cells_api_mock, "instance_update_from_api")
self.mox.StubOutWithMock(cells_rpcapi, "CellsAPI", use_mock_anything=True)
db.instance_get_by_uuid(ctxt, fake_uuid, columns_to_join=[]).AndReturn(old_ref)
db.instance_update_and_get_original(ctxt, fake_uuid, expected_updates, update_cells=False).AndReturn(
(old_ref, new_ref)
)
if cell_type == "api":
cells_rpcapi.CellsAPI().AndReturn(cells_api_mock)
cells_api_mock.instance_update_from_api(
ctxt, mox.IsA(instance.Instance), exp_vm_state, exp_task_state, admin_reset
)
elif cell_type == "compute":
cells_rpcapi.CellsAPI().AndReturn(cells_api_mock)
cells_api_mock.instance_update_at_top(ctxt, new_ref)
self.mox.ReplayAll()
inst = instance.Instance.get_by_uuid(ctxt, old_ref["uuid"])
self.assertEqual("old", inst.task_state)
self.assertEqual("old", inst.vm_state)
self.assertEqual("old", inst.user_data)
inst.vm_state = "meow"
inst.task_state = "wuff"
inst.user_data = "new"
inst.save(**save_kwargs)
self.assertEqual("newhost", inst.host)
self.assertEqual("meow", inst.vm_state)
self.assertEqual("wuff", inst.task_state)
self.assertEqual("new", inst.user_data)
self.assertEqual(set([]), inst.obj_what_changed())
def test_shelve(self):
CONF.shelved_offload_time = -1
db_instance = jsonutils.to_primitive(self._create_fake_instance())
self.compute.run_instance(self.context, instance=db_instance)
instance = instance_obj.Instance.get_by_uuid(
self.context, db_instance['uuid'],
expected_attrs=['metadata', 'system_metadata'])
image_id = 'fake_image_id'
host = 'fake-mini'
cur_time = timeutils.utcnow()
timeutils.set_time_override(cur_time)
instance.task_state = task_states.SHELVING
instance.save()
sys_meta = dict(instance.system_metadata)
sys_meta['shelved_at'] = timeutils.strtime(at=cur_time)
sys_meta['shelved_image_id'] = image_id
sys_meta['shelved_host'] = host
db_instance['system_metadata'] = utils.dict_to_metadata(sys_meta)
self.mox.StubOutWithMock(self.compute, '_notify_about_instance_usage')
self.mox.StubOutWithMock(self.compute.driver, 'snapshot')
self.mox.StubOutWithMock(self.compute.driver, 'power_off')
self.mox.StubOutWithMock(self.compute, '_get_power_state')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.compute._notify_about_instance_usage(self.context, instance,
'shelve.start')
self.compute.driver.power_off(instance)
self.compute._get_power_state(self.context,
instance).AndReturn(123)
self.compute.driver.snapshot(self.context, instance, 'fake_image_id',
mox.IgnoreArg())
db.instance_update_and_get_original(self.context, instance['uuid'],
{'power_state': 123,
'vm_state': vm_states.SHELVED,
'task_state': None,
'expected_task_state': [task_states.SHELVING,
task_states.SHELVING_IMAGE_UPLOADING],
'system_metadata': sys_meta},
update_cells=False,
columns_to_join=['metadata', 'system_metadata'],
).AndReturn((db_instance,
db_instance))
self.compute._notify_about_instance_usage(self.context,
instance, 'shelve.end')
self.mox.ReplayAll()
self.compute.shelve_instance(self.context, instance,
image_id=image_id)
self.mox.VerifyAll()
self.mox.UnsetStubs()
self.compute.terminate_instance(self.context, instance=instance)
def test_handle_schedule_error_adds_instance_fault(self):
instance = {"uuid": "fake-uuid"}
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.mox.StubOutWithMock(db, "instance_fault_create")
self.mox.StubOutWithMock(notifier, "notify")
db.instance_update_and_get_original(self.context, instance["uuid"], mox.IgnoreArg()).AndReturn((None, instance))
db.instance_fault_create(self.context, mox.IgnoreArg())
notifier.notify(self.context, mox.IgnoreArg(), "scheduler.run_instance", notifier.ERROR, mox.IgnoreArg())
self.mox.ReplayAll()
driver.handle_schedule_error(self.context, exception.NoValidHost("test"), instance["uuid"], {})
def _test_set_vm_state_and_notify(self, request_spec,
expected_uuids):
updates = dict(vm_state='fake-vm-state')
service = 'fake-service'
method = 'fake-method'
exc_info = 'exc_info'
self.mox.StubOutWithMock(compute_utils,
'add_instance_fault_from_exc')
self.mox.StubOutWithMock(notifications, 'send_update')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(rpc, 'get_notifier')
notifier = self.mox.CreateMockAnything()
rpc.get_notifier(service).AndReturn(notifier)
old_ref = 'old_ref'
new_ref = 'new_ref'
inst_obj = 'inst_obj'
for _uuid in expected_uuids:
db.instance_update_and_get_original(
self.context, _uuid, updates,
columns_to_join=['system_metadata']).AndReturn(
(old_ref, new_ref))
notifications.send_update(self.context, old_ref, inst_obj,
service=service)
compute_utils.add_instance_fault_from_exc(
self.context,
new_ref, exc_info, mox.IsA(tuple))
payload = dict(request_spec=request_spec,
instance_properties=request_spec.get(
'instance_properties', {}),
instance_id=_uuid,
state='fake-vm-state',
method=method,
reason=exc_info)
event_type = '%s.%s' % (service, method)
notifier.error(self.context, event_type, payload)
self.mox.ReplayAll()
with mock.patch.object(objects.Instance, '_from_db_object',
return_value=inst_obj):
scheduler_utils.set_vm_state_and_notify(self.context,
service,
method,
updates,
exc_info,
request_spec,
db)
def test_shelved_poll_not_timedout(self):
instance = self._create_fake_instance_obj()
sys_meta = instance.system_metadata
shelved_time = timeutils.utcnow()
timeutils.set_time_override(shelved_time)
timeutils.advance_time_seconds(CONF.shelved_offload_time - 1)
sys_meta['shelved_at'] = timeutils.strtime(at=shelved_time)
db.instance_update_and_get_original(self.context, instance['uuid'],
{'vm_state': vm_states.SHELVED, 'system_metadata': sys_meta})
self.mox.StubOutWithMock(self.compute.driver, 'destroy')
self.mox.ReplayAll()
self.compute._poll_shelved_instances(self.context)
def test_shelved_poll_not_timedout(self):
instance = jsonutils.to_primitive(self._create_fake_instance())
sys_meta = utils.metadata_to_dict(instance["system_metadata"])
shelved_time = timeutils.utcnow()
timeutils.set_time_override(shelved_time)
timeutils.advance_time_seconds(CONF.shelved_offload_time - 1)
sys_meta["shelved_at"] = timeutils.strtime(at=shelved_time)
db.instance_update_and_get_original(
self.context, instance["uuid"], {"vm_state": vm_states.SHELVED, "system_metadata": sys_meta}
)
self.mox.StubOutWithMock(self.compute.driver, "destroy")
self.mox.ReplayAll()
self.compute._poll_shelved_instances(self.context)
def test_handle_schedule_error_adds_instance_fault(self):
instance = {"uuid": "fake-uuid"}
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.mox.StubOutWithMock(db, "instance_fault_create")
db.instance_update_and_get_original(self.context, instance["uuid"], mox.IgnoreArg()).AndReturn((None, instance))
db.instance_fault_create(self.context, mox.IgnoreArg())
self.mox.StubOutWithMock(rpc, "get_notifier")
notifier = self.mox.CreateMockAnything()
rpc.get_notifier("conductor", CONF.host).AndReturn(notifier)
rpc.get_notifier("scheduler").AndReturn(notifier)
notifier.error(self.context, "scheduler.run_instance", mox.IgnoreArg())
self.mox.ReplayAll()
driver.handle_schedule_error(self.context, exception.NoValidHost("test"), instance["uuid"], {})
def test_set_vm_state_and_notify_adds_instance_fault(self):
request = {"instance_properties": {"uuid": "fake-uuid"}}
updates = {"vm_state": "foo"}
fake_inst = {"uuid": "fake-uuid"}
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.mox.StubOutWithMock(db, "instance_fault_create")
self.mox.StubOutWithMock(notifier, "notify")
db.instance_update_and_get_original(self.context, "fake-uuid", updates).AndReturn((None, fake_inst))
db.instance_fault_create(self.context, mox.IgnoreArg())
notifier.notify(self.context, mox.IgnoreArg(), "scheduler.foo", notifier.ERROR, mox.IgnoreArg())
self.mox.ReplayAll()
self.manager._set_vm_state_and_notify("foo", {"vm_state": "foo"}, self.context, None, request)
def test_basic_schedule_run_instance_no_hosts(self):
ctxt = context.RequestContext('fake', 'fake', False)
ctxt_elevated = 'fake-context-elevated'
fake_args = (1, 2, 3)
uuid = 'fake-uuid1'
instance_opts = {'fake_opt1': 'meow', 'launch_index': -1}
request_spec = {
'instance_uuids': [uuid],
'instance_properties': instance_opts
}
self.mox.StubOutWithMock(ctxt, 'elevated')
self.mox.StubOutWithMock(self.driver, 'hosts_up')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
# instance 1
ctxt.elevated().AndReturn(ctxt_elevated)
self.driver.hosts_up(ctxt_elevated, 'compute').AndReturn([])
old_ref, new_ref = db.instance_update_and_get_original(
ctxt, uuid, {
'vm_state': vm_states.ERROR,
'task_state': None
}).AndReturn(({}, {}))
compute_utils.add_instance_fault_from_exc(
ctxt, mox.IsA(conductor_api.LocalAPI), new_ref,
mox.IsA(exception.NoValidHost), mox.IgnoreArg())
self.mox.ReplayAll()
self.driver.schedule_run_instance(ctxt, request_spec, None, None, None,
None, {})
def test_run_instance_exception_puts_instance_in_error_state(self):
fake_instance_uuid = 'fake-instance-id'
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper('schedule_run_instance')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {
'instance_properties': inst,
'instance_uuids': [fake_instance_uuid]
}
self.manager.driver.schedule_run_instance(
self.context, request_spec, None, None, None, None, {},
False).AndRaise(exception.NoValidHost(reason=""))
old, new_ref = db.instance_update_and_get_original(
self.context, fake_instance_uuid, {
"vm_state": vm_states.ERROR,
"task_state": None
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), new_ref,
mox.IsA(exception.NoValidHost), mox.IgnoreArg())
self.mox.ReplayAll()
self.manager.run_instance(self.context, request_spec, None, None, None,
None, {}, False)
def save(self, context, expected_task_state=None):
"""Save updates to this instance
Column-wise updates will be made based on the result of
self.what_changed(). If expected_task_state is provided,
it will be checked against the in-database copy of the
instance before updates are made.
:param context: Security context
:param expected_task_state: Optional tuple of valid task states
for the instance to be in.
"""
updates = {}
changes = self.obj_what_changed()
for field in changes:
updates[field] = self[field]
if expected_task_state is not None:
updates['expected_task_state'] = expected_task_state
old_ref, inst_ref = db.instance_update_and_get_original(
context, self.uuid, updates)
expected_attrs = []
for attr in ('metadata', 'system_metadata'):
if hasattr(self, base.get_attrname(attr)):
expected_attrs.append(attr)
Instance._from_db_object(self, inst_ref, expected_attrs)
if 'vm_state' in changes or 'task_state' in changes:
notifications.send_update(context, old_ref, inst_ref)
self.obj_reset_changes()
def handle_schedule_error(context, ex, instance_uuid, request_spec):
"""On run_instance failure, update instance state and
send notifications.
"""
if not isinstance(ex, exception.NoValidHost):
LOG.exception(_("Exception during scheduler.run_instance"))
state = vm_states.ERROR.upper()
LOG.warning(_('Setting instance to %s state.'), state,
instance_uuid=instance_uuid)
(old_ref, new_ref) = db.instance_update_and_get_original(context,
instance_uuid, {'vm_state': vm_states.ERROR,
'task_state': None})
notifications.send_update(context, old_ref, new_ref,
service="scheduler")
compute_utils.add_instance_fault_from_exc(context,
new_ref, ex, sys.exc_info())
properties = request_spec.get('instance_properties', {})
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_states.ERROR,
method='run_instance',
reason=ex)
rpc.get_notifier('scheduler').error(context,
'scheduler.run_instance', payload)
def test_save(self):
ctxt = context.get_admin_context()
fake_inst = dict(self.fake_instance, host='oldhost')
fake_uuid = fake_inst['uuid']
self.mox.StubOutWithMock(db, 'instance_get_by_uuid')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(db, 'instance_info_cache_update')
db.instance_get_by_uuid(ctxt, fake_uuid, []).AndReturn(fake_inst)
db.instance_update_and_get_original(ctxt, fake_uuid, {
'user_data': 'foo'
}).AndReturn((fake_inst, dict(fake_inst, host='newhost')))
self.mox.ReplayAll()
inst = instance.Instance.get_by_uuid(ctxt, fake_uuid)
inst.user_data = 'foo'
inst.save()
self.assertEqual(inst.host, 'newhost')
def test_prep_resize_no_valid_host_back_in_shutoff_state(self):
fake_instance_uuid = 'fake-instance-id'
fake_instance = {'uuid': fake_instance_uuid, "vm_state": "stopped"}
inst = {"vm_state": "stopped", "task_state": ""}
self._mox_schedule_method_helper('select_destinations')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_type': 'fake_type',
'instance_uuids': [fake_instance_uuid],
'instance_properties': {'uuid': fake_instance_uuid}}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': fake_instance,
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.select_destinations(
self.context, request_spec, 'fake_props').AndRaise(
exception.NoValidHost(reason=""))
old_ref, new_ref = db.instance_update_and_get_original(self.context,
fake_instance_uuid,
{"vm_state": vm_states.STOPPED, "task_state": None}).AndReturn(
(inst, inst))
compute_utils.add_instance_fault_from_exc(self.context, new_ref,
mox.IsA(exception.NoValidHost), mox.IgnoreArg())
self.mox.ReplayAll()
self.manager.prep_resize(**kwargs)
def test_shelved_poll_not_timedout(self):
instance = jsonutils.to_primitive(self._create_fake_instance())
self.compute.run_instance(self.context, instance=instance)
sys_meta = utils.metadata_to_dict(instance['system_metadata'])
shelved_time = timeutils.utcnow()
timeutils.set_time_override(shelved_time)
timeutils.advance_time_seconds(CONF.shelved_offload_time - 1)
sys_meta['shelved_at'] = timeutils.strtime(at=shelved_time)
db.instance_update_and_get_original(self.context, instance['uuid'], {
'vm_state': vm_states.SHELVED,
'system_metadata': sys_meta
})
self.mox.StubOutWithMock(self.compute.driver, 'destroy')
self.mox.ReplayAll()
self.compute._poll_shelved_instances(self.context)
def test_run_instance_non_admin(self):
self.was_admin = False
def fake_get(context, *args, **kwargs):
# make sure this is called with admin context, even though
# we're using user context below
self.was_admin = context.is_admin
return {}
sched = fakes.FakeFilterScheduler()
self.stubs.Set(sched.host_manager, 'get_all_host_states', fake_get)
fake_context = context.RequestContext('user', 'project')
uuid = 'fake-uuid1'
instance_properties = {'project_id': 1, 'os_type': 'Linux'}
request_spec = {'instance_type': {'memory_mb': 1, 'local_gb': 1},
'instance_properties': instance_properties,
'instance_uuids': [uuid]}
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
old_ref, new_ref = db.instance_update_and_get_original(fake_context,
uuid, {'vm_state': vm_states.ERROR, 'task_state':
None}).AndReturn(({}, {}))
compute_utils.add_instance_fault_from_exc(fake_context,
mox.IsA(conductor_api.LocalAPI), new_ref,
mox.IsA(exception.NoValidHost), mox.IgnoreArg())
self.mox.ReplayAll()
sched.schedule_run_instance(
fake_context, request_spec, None, None, None, None, {})
self.assertTrue(self.was_admin)
def test_run_instance_no_hosts(self):
def _fake_empty_call_zone_method(*args, **kwargs):
return []
sched = fakes.FakeFilterScheduler()
uuid = 'fake-uuid1'
fake_context = context.RequestContext('user', 'project')
instance_properties = {'project_id': 1, 'os_type': 'Linux'}
request_spec = {'instance_type': {'memory_mb': 1, 'root_gb': 1,
'ephemeral_gb': 0},
'instance_properties': instance_properties,
'instance_uuids': [uuid]}
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
old_ref, new_ref = db.instance_update_and_get_original(fake_context,
uuid, {'vm_state': vm_states.ERROR, 'task_state':
None}).AndReturn(({}, {}))
compute_utils.add_instance_fault_from_exc(fake_context,
mox.IsA(conductor_api.LocalAPI), new_ref,
mox.IsA(exception.NoValidHost), mox.IgnoreArg())
self.mox.ReplayAll()
sched.schedule_run_instance(
fake_context, request_spec, None, None, None, None, {})
def handle_schedule_error(context, ex, instance_uuid, request_spec):
if not isinstance(ex, exception.NoValidHost):
LOG.exception(_("Exception during scheduler.run_instance"))
compute_utils.add_instance_fault_from_exc(context, instance_uuid, ex,
sys.exc_info())
state = vm_states.ERROR.upper()
LOG.warning(_('Setting instance to %(state)s state.'),
locals(),
instance_uuid=instance_uuid)
# update instance state and notify on the transition
(old_ref,
new_ref) = db.instance_update_and_get_original(context, instance_uuid, {
'vm_state': vm_states.ERROR,
'task_state': None
})
notifications.send_update(context, old_ref, new_ref, service="scheduler")
properties = request_spec.get('instance_properties', {})
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_states.ERROR,
method='run_instance',
reason=ex)
notifier.notify(context, notifier.publisher_id("scheduler"),
'scheduler.run_instance', notifier.ERROR, payload)
def test_shelved_poll_timedout(self):
active_instance = jsonutils.to_primitive(self._create_fake_instance())
self.compute.run_instance(self.context, instance=active_instance)
instance = jsonutils.to_primitive(self._create_fake_instance())
self.compute.run_instance(self.context, instance=instance)
sys_meta = utils.metadata_to_dict(instance['system_metadata'])
shelved_time = timeutils.utcnow()
timeutils.set_time_override(shelved_time)
timeutils.advance_time_seconds(CONF.shelved_offload_time + 1)
sys_meta['shelved_at'] = timeutils.strtime(at=shelved_time)
(old, instance) = db.instance_update_and_get_original(
self.context, instance['uuid'], {
'vm_state': vm_states.SHELVED,
'system_metadata': sys_meta
})
def fake_destroy(inst, nw_info, bdm):
# NOTE(alaski) There are too many differences between an instance
# as returned by instance_update_and_get_original and
# instance_get_all_by_filters so just compare the uuid.
self.assertEqual(instance['uuid'], inst['uuid'])
self.stubs.Set(self.compute.driver, 'destroy', fake_destroy)
self.compute._poll_shelved_instances(self.context)
def _test_set_vm_state_and_notify(self, request_spec,
expected_uuids):
updates = dict(vm_state='fake-vm-state')
service = 'fake-service'
method = 'fake-method'
exc_info = 'exc_info'
self.mox.StubOutWithMock(compute_utils,
'add_instance_fault_from_exc')
self.mox.StubOutWithMock(notifications, 'send_update')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(rpc, 'get_notifier')
notifier = self.mox.CreateMockAnything()
rpc.get_notifier(service).AndReturn(notifier)
old_ref = 'old_ref'
new_ref = 'new_ref'
for _uuid in expected_uuids:
db.instance_update_and_get_original(
self.context, _uuid, updates).AndReturn((old_ref, new_ref))
notifications.send_update(self.context, old_ref, new_ref,
service=service)
compute_utils.add_instance_fault_from_exc(
self.context,
new_ref, exc_info, mox.IsA(tuple))
payload = dict(request_spec=request_spec,
instance_properties=request_spec.get(
'instance_properties', {}),
instance_id=_uuid,
state='fake-vm-state',
method=method,
reason=exc_info)
event_type = '%s.%s' % (service, method)
notifier.error(self.context, event_type, payload)
self.mox.ReplayAll()
scheduler_utils.set_vm_state_and_notify(self.context,
service,
method,
updates,
exc_info,
request_spec,
db)
def test_handle_schedule_error_adds_instance_fault(self):
instance = {'uuid': 'fake-uuid'}
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(db, 'instance_fault_create')
self.mox.StubOutWithMock(notifier, 'notify')
db.instance_update_and_get_original(self.context, instance['uuid'],
mox.IgnoreArg()).AndReturn(
(None, instance))
db.instance_fault_create(self.context, mox.IgnoreArg())
notifier.notify(self.context, mox.IgnoreArg(),
'scheduler.run_instance', notifier.ERROR,
mox.IgnoreArg())
self.mox.ReplayAll()
driver.handle_schedule_error(self.context,
exception.NoValidHost('test'),
instance['uuid'], {})
def test_shelve_volume_backed(self):
db_instance = jsonutils.to_primitive(self._create_fake_instance())
self.compute.run_instance(self.context, instance=db_instance)
instance = instance_obj.Instance.get_by_uuid(
self.context, db_instance['uuid'],
expected_attrs=['metadata', 'system_metadata'])
instance.task_state = task_states.SHELVING
instance.save()
host = 'fake-mini'
cur_time = timeutils.utcnow()
timeutils.set_time_override(cur_time)
sys_meta = dict(instance.system_metadata)
sys_meta['shelved_at'] = timeutils.strtime(at=cur_time)
sys_meta['shelved_image_id'] = None
sys_meta['shelved_host'] = host
db_instance['system_metadata'] = utils.dict_to_metadata(sys_meta)
self.mox.StubOutWithMock(self.compute, '_notify_about_instance_usage')
self.mox.StubOutWithMock(self.compute.driver, 'power_off')
self.mox.StubOutWithMock(self.compute, '_get_power_state')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.compute._notify_about_instance_usage(self.context, instance,
'shelve_offload.start')
self.compute.driver.power_off(instance)
self.compute._get_power_state(self.context,
instance).AndReturn(123)
db.instance_update_and_get_original(self.context, instance['uuid'],
{'power_state': 123, 'host': None, 'node': None,
'vm_state': vm_states.SHELVED_OFFLOADED,
'task_state': None,
'expected_task_state': [task_states.SHELVING,
task_states.SHELVING_OFFLOADING]},
update_cells=False,
columns_to_join=['metadata', 'system_metadata'],
).AndReturn((db_instance, db_instance))
self.compute._notify_about_instance_usage(self.context, instance,
'shelve_offload.end')
self.mox.ReplayAll()
self.compute.shelve_offload_instance(self.context, instance)
self.mox.VerifyAll()
self.mox.UnsetStubs()
self.compute.terminate_instance(self.context, instance=instance)
def test_live_migration_basic(self):
"""Test basic schedule_live_migration functionality"""
self.mox.StubOutWithMock(self.driver, '_live_migration_src_check')
self.mox.StubOutWithMock(self.driver, '_live_migration_dest_check')
self.mox.StubOutWithMock(self.driver, '_live_migration_common_check')
self.mox.StubOutWithMock(self.driver.compute_rpcapi,
'check_can_live_migrate_destination')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(self.driver.compute_rpcapi, 'live_migration')
self.mox.StubOutWithMock(notifications, 'send_update')
dest = 'fake_host2'
block_migration = False
disk_over_commit = False
instance = jsonutils.to_primitive(self._live_migration_instance())
instance_id = instance['id']
instance_uuid = instance['uuid']
self.driver._live_migration_src_check(self.context, instance)
self.driver._live_migration_dest_check(self.context, instance, dest)
self.driver._live_migration_common_check(self.context, instance, dest)
self.driver.compute_rpcapi.check_can_live_migrate_destination(
self.context, instance, dest, block_migration,
disk_over_commit).AndReturn({})
db.instance_update_and_get_original(self.context, instance_uuid, {
"task_state": task_states.MIGRATING
}).AndReturn((instance, instance))
notifications.send_update(self.context,
instance,
instance,
service="scheduler")
self.driver.compute_rpcapi.live_migration(
self.context,
host=instance['host'],
instance=instance,
dest=dest,
block_migration=block_migration,
migrate_data={})
self.mox.ReplayAll()
self.driver.schedule_live_migration(self.context,
instance=instance,
dest=dest,
block_migration=block_migration,
disk_over_commit=disk_over_commit)
def test_send_on_task_change(self):
# pretend we just transitioned to task SPAWNING:
params = {"task_state": task_states.SPAWNING}
(old_ref, new_ref) = db.instance_update_and_get_original(
self.context, self.instance['uuid'], params)
notifications.send_update(self.context, old_ref, new_ref)
self.assertEqual(1, len(fake_notifier.NOTIFICATIONS))
def test_handle_schedule_error_adds_instance_fault(self):
instance = {'uuid': 'fake-uuid'}
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(db, 'instance_fault_create')
db.instance_update_and_get_original(self.context, instance['uuid'],
mox.IgnoreArg()).AndReturn(
(None, instance))
db.instance_fault_create(self.context, mox.IgnoreArg()).AndReturn(
test_instance_fault.fake_faults['fake-uuid'][0])
self.mox.StubOutWithMock(rpc, 'get_notifier')
notifier = self.mox.CreateMockAnything()
rpc.get_notifier('scheduler').AndReturn(notifier)
notifier.error(self.context, 'scheduler.run_instance', mox.IgnoreArg())
self.mox.ReplayAll()
driver.handle_schedule_error(self.context,
exception.NoValidHost('test'),
instance['uuid'], {})
def test_set_vm_state_and_notify_adds_instance_fault(self):
request = {'instance_properties': {'uuid': 'fake-uuid'}}
updates = {'vm_state': 'foo'}
fake_inst = {'uuid': 'fake-uuid'}
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(db, 'instance_fault_create')
self.mox.StubOutWithMock(notifier, 'notify')
db.instance_update_and_get_original(self.context, 'fake-uuid',
updates).AndReturn(
(None, fake_inst))
db.instance_fault_create(self.context, mox.IgnoreArg())
notifier.notify(self.context, mox.IgnoreArg(), 'scheduler.foo',
notifier.ERROR, mox.IgnoreArg())
self.mox.ReplayAll()
self.manager._set_vm_state_and_notify('foo', {'vm_state': 'foo'},
self.context, None, request)
def test_send_on_vm_change(self):
# pretend we just transitioned to ACTIVE:
params = {"vm_state": vm_states.ACTIVE}
(old_ref, new_ref) = db.instance_update_and_get_original(
self.context, self.instance['uuid'], params)
notifications.send_update(self.context, old_ref, new_ref)
self.assertEquals(1, len(test_notifier.NOTIFICATIONS))
def _test_set_vm_state_and_notify(self, request_spec, expected_uuids):
updates = dict(vm_state='fake-vm-state')
service = 'fake-service'
method = 'fake-method'
exc_info = 'exc_info'
publisher_id = 'fake-publisher-id'
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(notifications, 'send_update')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(notifier, 'publisher_id')
old_ref = 'old_ref'
new_ref = 'new_ref'
for uuid in expected_uuids:
db.instance_update_and_get_original(self.context, uuid,
updates).AndReturn(
(old_ref, new_ref))
notifications.send_update(self.context,
old_ref,
new_ref,
service=service)
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), new_ref,
exc_info, mox.IsA(tuple))
payload = dict(
request_spec=request_spec,
instance_properties=request_spec.get('instance_properties'),
instance_id=uuid,
state='fake-vm-state',
method=method,
reason=exc_info)
event_type = '%s.%s' % (service, method)
notifier.publisher_id(service).AndReturn(publisher_id)
notifier.notify(self.context, publisher_id, event_type,
notifier.ERROR, payload)
self.mox.ReplayAll()
scheduler_utils.set_vm_state_and_notify(self.context, service, method,
updates, exc_info,
request_spec, db)
def _set_instance_host(self, context, instance_uuid):
"""Tag the instance as belonging to this host. This should be done
while the COMPUTE_RESOURCES_SEMPAHORE is being held so the resource
claim will not be lost if the audit process starts.
"""
values = {'host': self.host, 'launched_on': self.host}
(old_ref, instance_ref) = db.instance_update_and_get_original(
context, instance_uuid, values)
notifications.send_update(context, old_ref, instance_ref)
return instance_ref
def test_instance_update_with_and_get_original(self):
ctxt = context.get_admin_context()
# Create an instance with some metadata
values = {'vm_state': 'building'}
instance = db.instance_create(ctxt, values)
(old_ref, new_ref) = db.instance_update_and_get_original(
ctxt, instance['uuid'], {'vm_state': 'needscoffee'})
self.assertEquals("building", old_ref["vm_state"])
self.assertEquals("needscoffee", new_ref["vm_state"])
def test_set_vm_state_and_notify_adds_instance_fault(self):
request = {'instance_properties': {'uuid': 'fake-uuid'}}
updates = {'vm_state': 'foo'}
fake_inst = {'uuid': 'fake-uuid'}
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(db, 'instance_fault_create')
self.mox.StubOutWithMock(rpc, 'get_notifier')
notifier = self.mox.CreateMockAnything()
rpc.get_notifier('scheduler').AndReturn(notifier)
db.instance_update_and_get_original(self.context, 'fake-uuid',
updates).AndReturn(
(None, fake_inst))
db.instance_fault_create(self.context, mox.IgnoreArg()).AndReturn(
test_instance_fault.fake_faults['fake-uuid'][0])
notifier.error(self.context, 'scheduler.foo', mox.IgnoreArg())
self.mox.ReplayAll()
self.manager._set_vm_state_and_notify('foo', {'vm_state': 'foo'},
self.context, None, request)
def _wait_for_boot():
try:
LOG.debug(_("Key is injected but instance is not running yet"),
instance=instance)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance['id'], {'vm_state': vm_states.BUILDING})
notifications.send_update(context, old_ref, new_ref)
state = self._conn.create_domain(xml_dict, bpath)
if state == power_state.RUNNING:
LOG.debug(_('instance %s: booted'),
instance['name'],
instance=instance)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance['id'],
{'vm_state': vm_states.ACTIVE})
notifications.send_update(context, old_ref, new_ref)
LOG.debug(_('~~~~~~ current state = %s ~~~~~~'),
state,
instance=instance)
LOG.debug(_("instance %s spawned successfully"),
instance['name'],
instance=instance)
else:
LOG.debug(_('instance %s:not booted'),
instance['name'],
instance=instance)
except Exception:
LOG.exception(_("Baremetal assignment is overcommitted."),
instance=instance)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance['id'], {
'vm_state': vm_states.ERROR,
'power_state': power_state.FAILED
})
notifications.send_update(context, old_ref, new_ref)
timer.stop()
def test_prep_resize_exception_host_in_error_state_and_raise(self):
"""Test that a NoValidHost exception for prep_resize puts
the instance in ACTIVE state
"""
fake_instance_uuid = 'fake-instance-id'
self._mox_schedule_method_helper('schedule_prep_resize')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_properties': {'uuid': fake_instance_uuid}}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': 'fake_instance',
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(
self.AnException('something happened'))
inst = {
"vm_state": "",
"task_state": "",
}
db.instance_update_and_get_original(self.context, fake_instance_uuid, {
"vm_state": vm_states.ERROR
}).AndReturn((inst, inst))
self.mox.ReplayAll()
# FIXME(comstud): Remove 'update_db' on future RPC version bump.
self.assertRaises(self.AnException,
self.manager.prep_resize,
update_db=False,
**kwargs)
def _set_vm_state_and_notify(self, method, updates, context, ex,
request_spec):
"""changes VM state and notifies."""
# FIXME(comstud): Re-factor this somehow. Not sure this belongs in the
# scheduler manager like this. We should make this easier.
# run_instance only sends a request_spec, and an instance may or may
# not have been created in the API (or scheduler) already. If it was
# created, there's a 'uuid' set in the instance_properties of the
# request_spec.
# (littleidea): I refactored this a bit, and I agree
# it should be easier :)
# The refactoring could go further but trying to minimize changes
# for essex timeframe
LOG.warning(_("Failed to schedule_%(method)s: %(ex)s") % locals())
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
# NOTE(vish): We shouldn't get here unless we have a catastrophic
# failure, so just set all instances to error. if uuid
# is not set, instance_uuids will be set to [None], this
# is solely to preserve existing behavior and can
# be removed along with the 'if instance_uuid:' if we can
# verify that uuid is always set.
uuids = [properties.get('uuid')]
for instance_uuid in request_spec.get('instance_uuids') or uuids:
if instance_uuid:
state = vm_state.upper()
LOG.warning(_('Setting instance to %(state)s state.'),
locals(),
instance_uuid=instance_uuid)
# update instance state and notify on the transition
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance_uuid, updates)
notifications.send_update(context,
old_ref,
new_ref,
service="scheduler")
compute_utils.add_instance_fault_from_exc(
context, new_ref, ex, sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
notifier.notify(context, notifier.publisher_id("scheduler"),
'scheduler.' + method, notifier.ERROR, payload)
def schedule_live_migration(self,
context,
instance_id,
dest,
block_migration=False,
disk_over_commit=False):
"""Live migration scheduling method.
:param context:
:param instance_id:
:param dest: destination host
:param block_migration: if true, block_migration.
:param disk_over_commit: if True, consider real(not virtual)
disk size.
:return:
The host where instance is running currently.
Then scheduler send request that host.
"""
# Whether instance exists and is running.
instance_ref = db.instance_get(context, instance_id)
# Checking instance.
self._live_migration_src_check(context, instance_ref)
# Checking destination host.
self._live_migration_dest_check(context, instance_ref, dest,
block_migration, disk_over_commit)
# Common checking.
self._live_migration_common_check(context, instance_ref, dest,
block_migration, disk_over_commit)
# Changing instance_state.
values = {"vm_state": vm_states.MIGRATING}
# update instance state and notify
(old_ref, new_instance_ref) = db.instance_update_and_get_original(
context, instance_id, values)
notifications.send_update(context,
old_ref,
new_instance_ref,
service="scheduler")
src = instance_ref['host']
cast_to_compute_host(context,
src,
'live_migration',
update_db=False,
instance_id=instance_id,
dest=dest,
block_migration=block_migration)
def test_save_rename_sends_notification(self):
# Tests that simply changing the 'display_name' on the instance
# will send a notification.
self.flags(enable=False, group='cells')
old_ref = dict(self.fake_instance, display_name='hello')
fake_uuid = old_ref['uuid']
expected_updates = dict(display_name='goodbye')
new_ref = dict(old_ref, **expected_updates)
self.mox.StubOutWithMock(db, 'instance_get_by_uuid')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(notifications, 'send_update')
db.instance_get_by_uuid(
self.context,
fake_uuid,
columns_to_join=['info_cache', 'security_groups'],
use_slave=False).AndReturn(old_ref)
db.instance_update_and_get_original(self.context,
fake_uuid,
expected_updates,
update_cells=False,
columns_to_join=[
'info_cache',
'security_groups',
'system_metadata'
]).AndReturn((old_ref, new_ref))
notifications.send_update(self.context, mox.IgnoreArg(),
mox.IgnoreArg())
self.mox.ReplayAll()
inst = instance.Instance.get_by_uuid(self.context,
old_ref['uuid'],
use_slave=False)
self.assertEqual('hello', inst.display_name)
inst.display_name = 'goodbye'
inst.save()
self.assertEqual('goodbye', inst.display_name)
self.assertEqual(set([]), inst.obj_what_changed())
def test_prep_resize_exception_host_in_error_state_and_raise(self):
fake_instance_uuid = 'fake-instance-id'
self._mox_schedule_method_helper('schedule_prep_resize')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_properties':
{'uuid': fake_instance_uuid}}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': 'fake_instance',
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(
test.TestingException('something happened'))
inst = {
"vm_state": "",
"task_state": "",
}
db.instance_update_and_get_original(self.context, fake_instance_uuid,
{"vm_state": vm_states.ERROR,
"task_state": None}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
fake_instance_uuid, mox.IsA(test.TestingException),
mox.IgnoreArg())
self.mox.ReplayAll()
self.assertRaises(test.TestingException, self.manager.prep_resize,
**kwargs)
def schedule_live_migration(self,
context,
dest,
block_migration=False,
disk_over_commit=False,
instance=None,
instance_id=None):
"""Live migration scheduling method.
:param context:
:param instance_id: (deprecated)
:param instance: instance dict
:param dest: destination host
:param block_migration: if true, block_migration.
:param disk_over_commit: if True, consider real(not virtual)
disk size.
:return:
The host where instance is running currently.
Then scheduler send request that host.
"""
# Check we can do live migration
if not instance:
instance = db.instance_get(context, instance_id)
self._live_migration_src_check(context, instance)
self._live_migration_dest_check(context, instance, dest)
self._live_migration_common_check(context, instance, dest)
migrate_data = self.compute_rpcapi.check_can_live_migrate_destination(
context, instance, dest, block_migration, disk_over_commit)
# Change instance_state
values = {"task_state": task_states.MIGRATING}
# update instance state and notify
(old_ref, new_instance_ref) = db.instance_update_and_get_original(
context, instance['uuid'], values)
notifications.send_update(context,
old_ref,
new_instance_ref,
service="scheduler")
# Perform migration
src = instance['host']
self.compute_rpcapi.live_migration(context,
host=src,
instance=new_instance_ref,
dest=dest,
block_migration=block_migration,
migrate_data=migrate_data)
