def handle_schedule_error(context, ex, instance_uuid, request_spec):
if not isinstance(ex, exception.NoValidHost):
LOG.exception(_("Exception during scheduler.run_instance"))
compute_utils.add_instance_fault_from_exc(context,
instance_uuid, ex, sys.exc_info())
state = vm_states.ERROR.upper()
LOG.warning(_('Setting instance to %(state)s state.'),
locals(), instance_uuid=instance_uuid)
# update instance state and notify on the transition
(old_ref, new_ref) = db.instance_update_and_get_original(context,
instance_uuid, {'vm_state': vm_states.ERROR,
'task_state': None})
notifications.send_update(context, old_ref, new_ref,
service="scheduler")
properties = request_spec.get('instance_properties', {})
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_states.ERROR,
method='run_instance',
reason=ex)
notifier.notify(context, notifier.publisher_id("scheduler"),
'scheduler.run_instance', notifier.ERROR, payload)
def test_run_instance_non_admin(self):
self.was_admin = False
def fake_get(context, *args, **kwargs):
# make sure this is called with admin context, even though
# we're using user context below
self.was_admin = context.is_admin
return {}
sched = fakes.FakeFilterScheduler()
self.stubs.Set(sched.host_manager, 'get_all_host_states', fake_get)
fake_context = context.RequestContext('user', 'project')
uuid = 'fake-uuid1'
instance_properties = {'project_id': 1, 'os_type': 'Linux'}
request_spec = {'instance_type': {'memory_mb': 1, 'local_gb': 1},
'instance_properties': instance_properties,
'instance_uuids': [uuid]}
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
compute_utils.add_instance_fault_from_exc(fake_context,
uuid, mox.IsA(exception.NoValidHost), mox.IgnoreArg())
db.instance_update_and_get_original(fake_context, uuid,
{'vm_state': vm_states.ERROR,
'task_state': None}).AndReturn(({}, {}))
self.mox.ReplayAll()
sched.schedule_run_instance(
fake_context, request_spec, None, None, None, None, {})
self.assertTrue(self.was_admin)
def handle_schedule_error(context, ex, instance_uuid, request_spec):
"""On run_instance failure, update instance state and
send notifications.
"""
if not isinstance(ex, exception.NoValidHost):
LOG.exception(_("Exception during scheduler.run_instance"))
state = vm_states.ERROR.upper()
LOG.warning(_("Setting instance to %s state."), state, instance_uuid=instance_uuid)
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance_uuid, {"vm_state": vm_states.ERROR, "task_state": None}
)
notifications.send_update(context, old_ref, new_ref, service="scheduler")
compute_utils.add_instance_fault_from_exc(context, conductor_api.LocalAPI(), new_ref, ex, sys.exc_info())
properties = request_spec.get("instance_properties", {})
payload = dict(
request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_states.ERROR,
method="run_instance",
reason=ex,
)
notifier.get_notifier("scheduler").error(context, "scheduler.run_instance", payload)
def set_vm_state_and_notify(context, service, method, updates, ex, request_spec, db):
"""changes VM state and notifies."""
LOG.warning(_("Failed to %(service)s_%(method)s: %(ex)s"), {"service": service, "method": method, "ex": ex})
vm_state = updates["vm_state"]
properties = request_spec.get("instance_properties", {})
# NOTE(vish): We shouldn't get here unless we have a catastrophic
# failure, so just set all instances to error. if uuid
# is not set, instance_uuids will be set to [None], this
# is solely to preserve existing behavior and can
# be removed along with the 'if instance_uuid:' if we can
# verify that uuid is always set.
uuids = [properties.get("uuid")]
notifier = rpc.get_notifier(service)
for instance_uuid in request_spec.get("instance_uuids") or uuids:
if instance_uuid:
state = vm_state.upper()
LOG.warning(_("Setting instance to %s state."), state, instance_uuid=instance_uuid)
# update instance state and notify on the transition
(old_ref, new_ref) = db.instance_update_and_get_original(context, instance_uuid, updates)
notifications.send_update(context, old_ref, new_ref, service=service)
compute_utils.add_instance_fault_from_exc(context, new_ref, ex, sys.exc_info())
payload = dict(
request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex,
)
event_type = "%s.%s" % (service, method)
notifier.error(context, event_type, payload)
def test_upload_image_retries_on_signal_exception(self):
self.flags(num_retries=2, group='glance')
params = self._get_upload_params()
self.mox.StubOutWithMock(self.session, 'call_plugin_serialized')
self.mox.StubOutWithMock(time, 'sleep')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
error_details = ["", "task signaled", "", ""]
error = self.session.XenAPI.Failure(details=error_details)
self.session.call_plugin_serialized('glance', 'upload_vhd2',
**params).AndRaise(error)
compute_utils.add_instance_fault_from_exc(self.context, self.instance,
error, (fake.Failure,
error,
mox.IgnoreArg()))
time.sleep(0.5)
# Note(johngarbutt) XenServer 6.1 and later has this error
error_details = ["", "signal: SIGTERM", "", ""]
error = self.session.XenAPI.Failure(details=error_details)
self.session.call_plugin_serialized('glance', 'upload_vhd2',
**params).AndRaise(error)
compute_utils.add_instance_fault_from_exc(self.context, self.instance,
error, (fake.Failure,
error,
mox.IgnoreArg()))
time.sleep(1)
self.session.call_plugin_serialized('glance', 'upload_vhd2',
**params)
self.mox.ReplayAll()
self.store.upload_image(self.context, self.session, self.instance,
'fake_image_uuid', ['fake_vdi_uuid'])
self.mox.VerifyAll()
def test_run_instance_no_hosts(self):
def _fake_empty_call_zone_method(*args, **kwargs):
return []
sched = fakes.FakeFilterScheduler()
uuid = 'fake-uuid1'
fake_context = context.RequestContext('user', 'project')
instance_properties = {'project_id': 1, 'os_type': 'Linux'}
request_spec = {'instance_type': {'memory_mb': 1, 'root_gb': 1,
'ephemeral_gb': 0},
'instance_properties': instance_properties,
'instance_uuids': [uuid]}
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
compute_utils.add_instance_fault_from_exc(fake_context,
uuid, mox.IsA(exception.NoValidHost), mox.IgnoreArg())
db.instance_update_and_get_original(fake_context, uuid,
{'vm_state': vm_states.ERROR,
'task_state': None}).AndReturn(({}, {}))
self.mox.ReplayAll()
sched.schedule_run_instance(
fake_context, request_spec, None, None, None, None, {})
def test_run_instance_non_admin(self):
self.was_admin = False
def fake_get(context, *args, **kwargs):
# make sure this is called with admin context, even though
# we're using user context below
self.was_admin = context.is_admin
return {}
sched = fakes.FakeFilterScheduler()
self.stubs.Set(sched.host_manager, "get_all_host_states", fake_get)
fake_context = context.RequestContext("user", "project")
uuid = "fake-uuid1"
instance_properties = {"project_id": 1, "os_type": "Linux"}
request_spec = {
"instance_type": {"memory_mb": 1, "local_gb": 1},
"instance_properties": instance_properties,
"instance_uuids": [uuid],
}
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
old_ref, new_ref = db.instance_update_and_get_original(
fake_context, uuid, {"vm_state": vm_states.ERROR, "task_state": None}
).AndReturn(({}, {}))
compute_utils.add_instance_fault_from_exc(
fake_context, mox.IsA(conductor_api.LocalAPI), new_ref, mox.IsA(exception.NoValidHost), mox.IgnoreArg()
)
self.mox.ReplayAll()
sched.schedule_run_instance(fake_context, request_spec, None, None, None, None, {}, False)
self.assertTrue(self.was_admin)
def set_vm_state_and_notify(context, instance_uuid, service, method, updates,
ex, request_spec):
"""changes VM state and notifies."""
LOG.warning(_LW("Failed to %(service)s_%(method)s: %(ex)s"),
{'service': service, 'method': method, 'ex': ex})
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
# NOTE(vish): We shouldn't get here unless we have a catastrophic
# failure, so just set the instance to its internal state
notifier = rpc.get_notifier(service)
state = vm_state.upper()
LOG.warning(_LW('Setting instance to %s state.'), state,
instance_uuid=instance_uuid)
instance = objects.Instance(context=context, uuid=instance_uuid,
**updates)
instance.obj_reset_changes(['uuid'])
instance.save()
compute_utils.add_instance_fault_from_exc(context,
instance, ex, sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
event_type = '%s.%s' % (service, method)
notifier.error(context, event_type, payload)
def test_basic_schedule_run_instance_no_hosts(self):
ctxt = context.RequestContext('fake', 'fake', False)
ctxt_elevated = 'fake-context-elevated'
fake_args = (1, 2, 3)
uuid = 'fake-uuid1'
instance_opts = {'fake_opt1': 'meow', 'launch_index': -1}
request_spec = {'instance_uuids': [uuid],
'instance_properties': instance_opts}
self.mox.StubOutWithMock(ctxt, 'elevated')
self.mox.StubOutWithMock(self.driver, 'hosts_up')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
# instance 1
ctxt.elevated().AndReturn(ctxt_elevated)
self.driver.hosts_up(ctxt_elevated, 'compute').AndReturn([])
old_ref, new_ref = db.instance_update_and_get_original(ctxt, uuid,
{'vm_state': vm_states.ERROR,
'task_state': None}).AndReturn(({}, {}))
compute_utils.add_instance_fault_from_exc(ctxt,
new_ref, mox.IsA(exception.NoValidHost), mox.IgnoreArg())
self.mox.ReplayAll()
self.driver.schedule_run_instance(
ctxt, request_spec, None, None, None, None, {})
def test_live_migration_set_vmstate_error(self):
inst = {"uuid": "fake-instance-id", "vm_state": vm_states.ACTIVE}
dest = "fake_host"
block_migration = False
disk_over_commit = False
self._mox_schedule_method_helper("schedule_live_migration")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.manager.driver.schedule_live_migration(
self.context, inst, dest, block_migration, disk_over_commit
).AndRaise(ValueError)
db.instance_update_and_get_original(self.context, inst["uuid"], {"vm_state": vm_states.ERROR}).AndReturn(
(inst, inst)
)
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), inst, mox.IsA(ValueError), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.stub_out_client_exceptions()
self.assertRaises(
ValueError, self.manager.live_migration, self.context, inst, dest, block_migration, disk_over_commit
)
def test_live_migration_schedule_novalidhost(self):
inst = {"uuid": "fake-instance-id", "vm_state": vm_states.ACTIVE, "task_state": task_states.MIGRATING}
dest = None
block_migration = False
disk_over_commit = False
self._mox_schedule_method_helper("schedule_live_migration")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.manager.driver.schedule_live_migration(
self.context, inst, dest, block_migration, disk_over_commit
).AndRaise(exception.NoValidHost(reason=""))
db.instance_update_and_get_original(
self.context,
inst["uuid"],
{"vm_state": inst["vm_state"], "task_state": None, "expected_task_state": task_states.MIGRATING},
).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), inst, mox.IsA(exception.NoValidHost), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.stub_out_client_exceptions()
self.assertRaises(
exception.NoValidHost,
self.manager.live_migration,
self.context,
inst,
dest,
block_migration,
disk_over_commit,
)
def test_live_migration_schedule_novalidhost(self):
inst = {"uuid": "fake-instance-id",
"vm_state": vm_states.ACTIVE,
"task_state": task_states.MIGRATING, }
dest = None
block_migration = False
disk_over_commit = False
pclm = "pclm"
self.mox.StubOutWithMock(self.manager, '_schedule_live_migration')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.manager._schedule_live_migration(self.context,
inst, dest, block_migration, disk_over_commit, pclm).AndRaise(
exception.NoValidHost(reason=""))
db.instance_update_and_get_original(self.context, inst["uuid"],
{"vm_state": inst['vm_state'],
"task_state": None,
"expected_task_state": task_states.MIGRATING,
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
mox.IsA(conductor_api.LocalAPI), inst,
mox.IsA(exception.NoValidHost),
mox.IgnoreArg())
self.mox.ReplayAll()
self.manager = utils.ExceptionHelper(self.manager)
self.assertRaises(exception.NoValidHost,
self.manager.live_migration,
self.context, inst, dest, block_migration,
disk_over_commit, pclm)
def test_live_migration_set_vmstate_error(self):
inst = {"uuid": "fake-instance-id",
"vm_state": vm_states.ACTIVE, }
dest = 'fake_host'
block_migration = False
disk_over_commit = False
pclm = "pclm"
self.mox.StubOutWithMock(self.manager, '_schedule_live_migration')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.manager._schedule_live_migration(self.context,
inst, dest, block_migration, disk_over_commit, pclm).AndRaise(
ValueError)
db.instance_update_and_get_original(self.context, inst["uuid"],
{"vm_state": vm_states.ERROR,
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
mox.IsA(conductor_api.LocalAPI), inst,
mox.IsA(ValueError),
mox.IgnoreArg())
self.mox.ReplayAll()
self.manager = utils.ExceptionHelper(self.manager)
self.assertRaises(ValueError,
self.manager.live_migration,
self.context, inst, dest, block_migration,
disk_over_commit, pclm)
def test_prep_resize_exception_host_in_error_state_and_raise(self):
fake_instance_uuid = "fake-instance-id"
fake_instance = {"uuid": fake_instance_uuid}
self._mox_schedule_method_helper("schedule_prep_resize")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
request_spec = {"instance_properties": {"uuid": fake_instance_uuid}}
kwargs = {
"context": self.context,
"image": "fake_image",
"request_spec": request_spec,
"filter_properties": "fake_props",
"instance": fake_instance,
"instance_type": "fake_type",
"reservations": list("fake_res"),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(test.TestingException("something happened"))
inst = {"vm_state": "", "task_state": ""}
old_ref, new_ref = db.instance_update_and_get_original(
self.context, fake_instance_uuid, {"vm_state": vm_states.ERROR, "task_state": None}
).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), new_ref, mox.IsA(test.TestingException), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.assertRaises(test.TestingException, self.manager.prep_resize, **kwargs)
def test_prep_resize_no_valid_host_back_in_active_state(self):
fake_instance_uuid = "fake-instance-id"
fake_instance = {"uuid": fake_instance_uuid}
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper("schedule_prep_resize")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
request_spec = {
"instance_type": "fake_type",
"instance_uuids": [fake_instance_uuid],
"instance_properties": {"uuid": fake_instance_uuid},
}
kwargs = {
"context": self.context,
"image": "fake_image",
"request_spec": request_spec,
"filter_properties": "fake_props",
"instance": fake_instance,
"instance_type": "fake_type",
"reservations": list("fake_res"),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(exception.NoValidHost(reason=""))
old_ref, new_ref = db.instance_update_and_get_original(
self.context, fake_instance_uuid, {"vm_state": vm_states.ACTIVE, "task_state": None}
).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), new_ref, mox.IsA(exception.NoValidHost), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.manager.prep_resize(**kwargs)
def test_live_migration_compute_service_notavailable(self):
inst = {"uuid": "fake-instance-id",
"vm_state": vm_states.ACTIVE,
"task_state": task_states.MIGRATING, }
dest = 'fake_host'
block_migration = False
disk_over_commit = False
self._mox_schedule_method_helper('schedule_live_migration')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.manager.driver.schedule_live_migration(self.context,
inst, dest, block_migration, disk_over_commit).AndRaise(
exception.ComputeServiceUnavailable(host="src"))
db.instance_update_and_get_original(self.context, inst["uuid"],
{"vm_state": inst['vm_state'],
"task_state": None,
"expected_task_state": task_states.MIGRATING,
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
mox.IsA(conductor_api.LocalAPI), inst,
mox.IsA(exception.ComputeServiceUnavailable),
mox.IgnoreArg())
self.mox.ReplayAll()
self.stub_out_client_exceptions()
self.assertRaises(exception.ComputeServiceUnavailable,
self.manager.live_migration,
self.context, inst, dest, block_migration,
disk_over_commit)
def test_run_instance_exception_puts_instance_in_error_state(self):
fake_instance_uuid = 'fake-instance-id'
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper('schedule_run_instance')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_properties': inst,
'instance_uuids': [fake_instance_uuid]}
self.manager.driver.schedule_run_instance(self.context,
request_spec, None, None, None, None, {}).AndRaise(
exception.NoValidHost(reason=""))
old, new_ref = db.instance_update_and_get_original(self.context,
fake_instance_uuid,
{"vm_state": vm_states.ERROR,
"task_state": None}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
mox.IsA(conductor_api.LocalAPI), new_ref,
mox.IsA(exception.NoValidHost), mox.IgnoreArg())
self.mox.ReplayAll()
self.manager.run_instance(self.context, request_spec,
None, None, None, None, {})
def test_prep_resize_no_valid_host_back_in_active_state(self):
fake_instance_uuid = 'fake-instance-id'
fake_instance = {'uuid': fake_instance_uuid}
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper('schedule_prep_resize')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_type': 'fake_type',
'instance_uuids': [fake_instance_uuid],
'instance_properties': {'uuid': fake_instance_uuid}}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': fake_instance,
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(
exception.NoValidHost(reason=""))
old_ref, new_ref = db.instance_update_and_get_original(self.context,
fake_instance_uuid,
{"vm_state": vm_states.ACTIVE, "task_state": None}).AndReturn(
(inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
mox.IsA(conductor_api.LocalAPI), new_ref,
mox.IsA(exception.NoValidHost), mox.IgnoreArg())
self.mox.ReplayAll()
self.manager.prep_resize(**kwargs)
def test_live_migration_compute_service_notavailable(self):
inst = {"uuid": "fake-instance-id", "vm_state": vm_states.ACTIVE, "task_state": task_states.MIGRATING}
dest = "fake_host"
block_migration = False
disk_over_commit = False
self.mox.StubOutWithMock(self.manager, "_schedule_live_migration")
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
self.manager._schedule_live_migration(self.context, inst, dest, block_migration, disk_over_commit).AndRaise(
exception.ComputeServiceUnavailable(host="src")
)
db.instance_update_and_get_original(
self.context,
inst["uuid"],
{"vm_state": inst["vm_state"], "task_state": None, "expected_task_state": task_states.MIGRATING},
).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, inst, mox.IsA(exception.ComputeServiceUnavailable), mox.IgnoreArg()
)
self.mox.ReplayAll()
self.manager = utils.ExceptionHelper(self.manager)
self.assertRaises(
exception.ComputeServiceUnavailable,
self.manager.live_migration,
self.context,
inst,
dest,
block_migration,
disk_over_commit,
)
def test_run_instance_no_hosts(self):
def _fake_empty_call_zone_method(*args, **kwargs):
return []
sched = fakes.FakeFilterScheduler()
uuid = "fake-uuid1"
fake_context = context.RequestContext("user", "project")
instance_properties = {"project_id": 1, "os_type": "Linux"}
request_spec = {
"instance_type": {"memory_mb": 1, "root_gb": 1, "ephemeral_gb": 0},
"instance_properties": instance_properties,
"instance_uuids": [uuid],
}
self.mox.StubOutWithMock(compute_utils, "add_instance_fault_from_exc")
self.mox.StubOutWithMock(db, "instance_update_and_get_original")
old_ref, new_ref = db.instance_update_and_get_original(
fake_context, uuid, {"vm_state": vm_states.ERROR, "task_state": None}
).AndReturn(({}, {}))
compute_utils.add_instance_fault_from_exc(
fake_context, mox.IsA(conductor_api.LocalAPI), new_ref, mox.IsA(exception.NoValidHost), mox.IgnoreArg()
)
self.mox.StubOutWithMock(db, "compute_node_get_all")
db.compute_node_get_all(mox.IgnoreArg()).AndReturn([])
self.mox.ReplayAll()
sched.schedule_run_instance(fake_context, request_spec, None, None, None, None, {}, False)
def handle_schedule_error(context, ex, instance_uuid, request_spec):
"""On run_instance failure, update instance state and
send notifications.
"""
if isinstance(ex, exception.NoValidHost):
LOG.warning(_LW("NoValidHost exception with message: \'%s\'"),
ex.format_message().strip(),
instance_uuid=instance_uuid)
else:
LOG.exception(_LE("Exception during scheduler.run_instance"))
state = vm_states.ERROR.upper()
LOG.warning(_LW('Setting instance to %s state.'), state,
instance_uuid=instance_uuid)
(old_ref, new_ref) = db.instance_update_and_get_original(context,
instance_uuid, {'vm_state': vm_states.ERROR,
'task_state': None})
notifications.send_update(context, old_ref, new_ref,
service="scheduler")
compute_utils.add_instance_fault_from_exc(context,
new_ref, ex, sys.exc_info())
properties = request_spec.get('instance_properties', {})
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_states.ERROR,
method='run_instance',
reason=ex)
rpc.get_notifier('scheduler').error(context,
'scheduler.run_instance', payload)
def _add_instance_fault(self, error, exc_info):
LOG.warning("Ignoring error while configuring instance with agent: %s",
error, instance=self.instance, exc_info=True)
try:
ctxt = context.get_admin_context()
compute_utils.add_instance_fault_from_exc(
ctxt, self.instance, error, exc_info=exc_info)
except Exception:
LOG.debug("Error setting instance fault.", exc_info=True)
def _add_instance_fault(self, error, exc_info):
LOG.warning(_("Ignoring error while configuring instance with "
"agent: %s") % error,
instance=self.instance, exc_info=True)
try:
ctxt = context.get_admin_context()
capi = conductor.API()
compute_utils.add_instance_fault_from_exc(
ctxt, capi, self.instance, error, exc_info=exc_info)
except Exception:
pass
def _test_set_vm_state_and_notify(self, request_spec,
expected_uuids):
updates = dict(vm_state='fake-vm-state')
service = 'fake-service'
method = 'fake-method'
exc_info = 'exc_info'
self.mox.StubOutWithMock(compute_utils,
'add_instance_fault_from_exc')
self.mox.StubOutWithMock(notifications, 'send_update')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(rpc, 'get_notifier')
notifier = self.mox.CreateMockAnything()
rpc.get_notifier(service).AndReturn(notifier)
old_ref = 'old_ref'
new_ref = 'new_ref'
inst_obj = 'inst_obj'
for _uuid in expected_uuids:
db.instance_update_and_get_original(
self.context, _uuid, updates,
columns_to_join=['system_metadata']).AndReturn(
(old_ref, new_ref))
notifications.send_update(self.context, old_ref, inst_obj,
service=service)
compute_utils.add_instance_fault_from_exc(
self.context,
new_ref, exc_info, mox.IsA(tuple))
payload = dict(request_spec=request_spec,
instance_properties=request_spec.get(
'instance_properties', {}),
instance_id=_uuid,
state='fake-vm-state',
method=method,
reason=exc_info)
event_type = '%s.%s' % (service, method)
notifier.error(self.context, event_type, payload)
self.mox.ReplayAll()
with mock.patch.object(objects.Instance, '_from_db_object',
return_value=inst_obj):
scheduler_utils.set_vm_state_and_notify(self.context,
service,
method,
updates,
exc_info,
request_spec,
db)
def set_vm_state_and_notify(context, instance_uuid, service, method, updates,
ex, request_spec):
"""Updates the instance, sets the fault and sends an error notification.
:param context: The request context.
:param instance_uuid: The UUID of the instance to update.
:param service: The name of the originating service, e.g. 'compute_task'.
This becomes part of the publisher_id for the notification payload.
:param method: The method that failed, e.g. 'migrate_server'.
:param updates: dict of updates for the instance object, typically a
vm_state and/or task_state value.
:param ex: An exception which occurred during the given method.
:param request_spec: Optional request spec.
"""
# e.g. "Failed to compute_task_migrate_server: No valid host was found"
LOG.warning("Failed to %(service)s_%(method)s: %(ex)s",
{'service': service, 'method': method, 'ex': ex})
# Convert the request spec to a dict if needed.
if request_spec is not None:
if isinstance(request_spec, objects.RequestSpec):
request_spec = request_spec.to_legacy_request_spec_dict()
else:
request_spec = {}
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
notifier = rpc.get_notifier(service)
state = vm_state.upper()
LOG.warning('Setting instance to %s state.', state,
instance_uuid=instance_uuid)
instance = objects.Instance(context=context, uuid=instance_uuid,
**updates)
instance.obj_reset_changes(['uuid'])
instance.save()
compute_utils.add_instance_fault_from_exc(
context, instance, ex, sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
event_type = '%s.%s' % (service, method)
notifier.error(context, event_type, payload)
compute_utils.notify_about_compute_task_error(
context, method, instance_uuid, request_spec, vm_state, ex,
traceback.format_exc())
def _set_vm_state_and_notify(self, method, updates, context, ex,
request_spec):
"""changes VM state and notifies."""
# FIXME(comstud): Re-factor this somehow. Not sure this belongs in the
# scheduler manager like this. We should make this easier.
# run_instance only sends a request_spec, and an instance may or may
# not have been created in the API (or scheduler) already. If it was
# created, there's a 'uuid' set in the instance_properties of the
# request_spec.
# (littleidea): I refactored this a bit, and I agree
# it should be easier :)
# The refactoring could go further but trying to minimize changes
# for essex timeframe
LOG.warning(_("Failed to schedule_%(method)s: %(ex)s"),
{'method': method, 'ex': ex})
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
# NOTE(vish): We shouldn't get here unless we have a catastrophic
# failure, so just set all instances to error. if uuid
# is not set, instance_uuids will be set to [None], this
# is solely to preserve existing behavior and can
# be removed along with the 'if instance_uuid:' if we can
# verify that uuid is always set.
uuids = [properties.get('uuid')]
for instance_uuid in request_spec.get('instance_uuids') or uuids:
if instance_uuid:
state = vm_state.upper()
LOG.warning(_('Setting instance to %s state.'), state,
instance_uuid=instance_uuid)
# update instance state and notify on the transition
(old_ref, new_ref) = self.db.instance_update_and_get_original(
context, instance_uuid, updates)
notifications.send_update(context, old_ref, new_ref,
service="scheduler")
compute_utils.add_instance_fault_from_exc(context,
conductor_api.LocalAPI(),
new_ref, ex, sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
notifier.notify(context, notifier.publisher_id("scheduler"),
'scheduler.' + method, notifier.ERROR, payload)
def set_vm_state_and_notify(context, instance_uuid, service, method, updates,
ex, request_spec):
"""Updates the instance, sets the fault and sends an error notification.
:param context: The request context.
:param instance_uuid: The UUID of the instance to update.
:param service: The name of the originating service, e.g. 'compute_task'.
This becomes part of the publisher_id for the notification payload.
:param method: The method that failed, e.g. 'migrate_server'.
:param updates: dict of updates for the instance object, typically a
vm_state and/or task_state value.
:param ex: An exception which occurred during the given method.
:param request_spec: Optional request spec.
"""
# e.g. "Failed to compute_task_migrate_server: No valid host was found"
LOG.warning("Failed to %(service)s_%(method)s: %(ex)s",
{'service': service, 'method': method, 'ex': ex})
# Convert the request spec to a dict if needed.
if request_spec is not None:
if isinstance(request_spec, objects.RequestSpec):
request_spec = request_spec.to_legacy_request_spec_dict()
else:
request_spec = {}
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
notifier = rpc.get_notifier(service)
state = vm_state.upper()
LOG.warning('Setting instance to %s state.', state,
instance_uuid=instance_uuid)
instance = objects.Instance(context=context, uuid=instance_uuid,
**updates)
instance.obj_reset_changes(['uuid'])
instance.save()
compute_utils.add_instance_fault_from_exc(
context, instance, ex, sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
event_type = '%s.%s' % (service, method)
# TODO(mriedem): Send a versioned notification.
notifier.error(context, event_type, payload)
def _test_set_vm_state_and_notify(self, request_spec,
expected_uuids):
updates = dict(vm_state='fake-vm-state')
service = 'fake-service'
method = 'fake-method'
exc_info = 'exc_info'
self.mox.StubOutWithMock(compute_utils,
'add_instance_fault_from_exc')
self.mox.StubOutWithMock(notifications, 'send_update')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
self.mox.StubOutWithMock(notify, 'get_notifier')
notifier = self.mox.CreateMockAnything()
notify.get_notifier('conductor', CONF.host).AndReturn(notifier)
notify.get_notifier(service).AndReturn(notifier)
old_ref = 'old_ref'
new_ref = 'new_ref'
for uuid in expected_uuids:
db.instance_update_and_get_original(
self.context, uuid, updates).AndReturn((old_ref, new_ref))
notifications.send_update(self.context, old_ref, new_ref,
service=service)
compute_utils.add_instance_fault_from_exc(
self.context,
mox.IsA(conductor_api.LocalAPI),
new_ref, exc_info, mox.IsA(tuple))
payload = dict(request_spec=request_spec,
instance_properties=request_spec.get(
'instance_properties', {}),
instance_id=uuid,
state='fake-vm-state',
method=method,
reason=exc_info)
event_type = '%s.%s' % (service, method)
notifier.error(self.context, event_type, payload)
self.mox.ReplayAll()
scheduler_utils.set_vm_state_and_notify(self.context,
service,
method,
updates,
exc_info,
request_spec,
db)
def _set_vm_state_and_notify(self, method, updates, context, ex,
request_spec):
"""changes VM state and notifies."""
# FIXME(comstud): Re-factor this somehow. Not sure this belongs in the
# scheduler manager like this. We should make this easier.
# run_instance only sends a request_spec, and an instance may or may
# not have been created in the API (or scheduler) already. If it was
# created, there's a 'uuid' set in the instance_properties of the
# request_spec.
# (littleidea): I refactored this a bit, and I agree
# it should be easier :)
# The refactoring could go further but trying to minimize changes
# for essex timeframe
LOG.warning(_("Failed to schedule_%(method)s: %(ex)s") % locals())
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
# NOTE(vish): We shouldn't get here unless we have a catastrophic
# failure, so just set all instances to error. if uuid
# is not set, instance_uuids will be set to [None], this
# is solely to preserve existing behavior and can
# be removed along with the 'if instance_uuid:' if we can
# verify that uuid is always set.
uuids = [properties.get('uuid')]
for instance_uuid in request_spec.get('instance_uuids') or uuids:
if instance_uuid:
state = vm_state.upper()
LOG.warning(_('Setting instance to %(state)s state.'),
locals(), instance_uuid=instance_uuid)
# update instance state and notify on the transition
(old_ref, new_ref) = self.db.instance_update_and_get_original(
context, instance_uuid, updates)
notifications.send_update(context, old_ref, new_ref,
service="scheduler")
compute_utils.add_instance_fault_from_exc(context,
conductor_api.LocalAPI(),
new_ref, ex, sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
notifier.notify(context, notifier.publisher_id("scheduler"),
'scheduler.' + method, notifier.ERROR, payload)
def _add_instance_fault(self, error, exc_info):
LOG.warning(_("Ignoring error while configuring instance with "
"agent: %s") % error,
instance=self.instance,
exc_info=True)
try:
ctxt = context.get_admin_context()
capi = conductor.API()
compute_utils.add_instance_fault_from_exc(ctxt,
capi,
self.instance,
error,
exc_info=exc_info)
except Exception:
pass
def set_vm_state_and_notify(context, service, method, updates, ex,
request_spec, db):
"""changes VM state and notifies."""
LOG.warning(_("Failed to %(service)s_%(method)s: %(ex)s"), {
'service': service,
'method': method,
'ex': ex
})
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
# NOTE(vish): We shouldn't get here unless we have a catastrophic
# failure, so just set all instances to error. if uuid
# is not set, instance_uuids will be set to [None], this
# is solely to preserve existing behavior and can
# be removed along with the 'if instance_uuid:' if we can
# verify that uuid is always set.
uuids = [properties.get('uuid')]
from nova.conductor import api as conductor_api
conductor = conductor_api.LocalAPI()
notifier = rpc.get_notifier(service)
for instance_uuid in request_spec.get('instance_uuids') or uuids:
if instance_uuid:
state = vm_state.upper()
LOG.warning(_('Setting instance to %s state.'),
state,
instance_uuid=instance_uuid)
# update instance state and notify on the transition
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance_uuid, updates)
notifications.send_update(context,
old_ref,
new_ref,
service=service)
compute_utils.add_instance_fault_from_exc(context, conductor,
new_ref, ex,
sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
event_type = '%s.%s' % (service, method)
notifier.error(context, event_type, payload)
def set_vm_state_and_notify(context, service, method, updates, ex,
request_spec, db):
"""changes VM state and notifies."""
LOG.warning(_LW("Failed to %(service)s_%(method)s: %(ex)s"),
{'service': service, 'method': method, 'ex': ex})
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
# NOTE(vish): We shouldn't get here unless we have a catastrophic
# failure, so just set all instances to error. if uuid
# is not set, instance_uuids will be set to [None], this
# is solely to preserve existing behavior and can
# be removed along with the 'if instance_uuid:' if we can
# verify that uuid is always set.
uuids = [properties.get('uuid')]
notifier = rpc.get_notifier(service)
for instance_uuid in request_spec.get('instance_uuids') or uuids:
if instance_uuid:
state = vm_state.upper()
LOG.warning(_LW('Setting instance to %s state.'), state,
instance_uuid=instance_uuid)
# update instance state and notify on the transition
# NOTE(hanlind): the send_update() call below is going to want to
# know about the flavor, so we need to join the appropriate things
# here and objectify the results.
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance_uuid, updates,
columns_to_join=['system_metadata'])
inst_obj = objects.Instance._from_db_object(
context, objects.Instance(), new_ref,
expected_attrs=['system_metadata'])
notifications.send_update(context, old_ref, inst_obj,
service=service)
compute_utils.add_instance_fault_from_exc(context,
new_ref, ex, sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
event_type = '%s.%s' % (service, method)
notifier.error(context, event_type, payload)
def set_vm_state_and_notify(context, service, method, updates, ex,
request_spec, db):
"""changes VM state and notifies."""
LOG.warning(_LW("Failed to %(service)s_%(method)s: %(ex)s"),
{'service': service, 'method': method, 'ex': ex})
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
# NOTE(vish): We shouldn't get here unless we have a catastrophic
# failure, so just set all instances to error. if uuid
# is not set, instance_uuids will be set to [None], this
# is solely to preserve existing behavior and can
# be removed along with the 'if instance_uuid:' if we can
# verify that uuid is always set.
uuids = [properties.get('uuid')]
notifier = rpc.get_notifier(service)
for instance_uuid in request_spec.get('instance_uuids') or uuids:
if instance_uuid:
state = vm_state.upper()
LOG.warning(_LW('Setting instance to %s state.'), state,
instance_uuid=instance_uuid)
# update instance state and notify on the transition
# NOTE(hanlind): the send_update() call below is going to want to
# know about the flavor, so we need to join the appropriate things
# here and objectify the results.
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance_uuid, updates,
columns_to_join=['system_metadata'])
inst_obj = objects.Instance._from_db_object(
context, objects.Instance(), new_ref,
expected_attrs=['system_metadata'])
notifications.send_update(context, old_ref, inst_obj,
service=service)
compute_utils.add_instance_fault_from_exc(context,
new_ref, ex, sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
event_type = '%s.%s' % (service, method)
notifier.error(context, event_type, payload)
def test_prep_resize_exception_host_in_error_state_and_raise(self):
fake_instance_uuid = 'fake-instance-id'
fake_instance = {'uuid': fake_instance_uuid}
self._mox_schedule_method_helper('select_destinations')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {
'instance_properties': {
'uuid': fake_instance_uuid
},
'instance_uuids': [fake_instance_uuid]
}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': fake_instance,
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.select_destinations(
self.context, request_spec,
'fake_props').AndRaise(test.TestingException('something happened'))
inst = {
"vm_state": "",
"task_state": "",
}
old_ref, new_ref = db.instance_update_and_get_original(
self.context, fake_instance_uuid, {
"vm_state": vm_states.ERROR,
"task_state": None
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), new_ref,
mox.IsA(test.TestingException), mox.IgnoreArg())
self.mox.ReplayAll()
self.assertRaises(test.TestingException, self.manager.prep_resize,
**kwargs)
def handle_schedule_error(context, ex, instance_uuid, request_spec, retry):
"""On run_instance failure, update instance state and
send notifications.
"""
if isinstance(ex, exception.NoValidHost):
LOG.warning(_LW("NoValidHost exception with message: \'%s\'"),
ex.format_message().strip(),
instance_uuid=instance_uuid)
else:
LOG.exception(_("Exception during scheduler.run_instance"))
state = vm_states.ERROR.upper()
LOG.warning(_LW('Setting instance to %s state.'),
state,
instance_uuid=instance_uuid)
(old_ref,
new_ref) = db.instance_update_and_get_original(context, instance_uuid, {
'vm_state': vm_states.ERROR,
'task_state': None
})
notifications.send_update(context, old_ref, new_ref, service="scheduler")
# PF9 change
# Use refined version instead of generic NoHostFound in the end
if retry and 'pf9_error' in retry:
compute_utils.add_instance_fault_from_pf9_error(
context, new_ref, retry['pf9_error'])
else:
compute_utils.add_instance_fault_from_exc(context, new_ref, ex,
sys.exc_info())
# PF9 end
properties = request_spec.get('instance_properties', {})
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_states.ERROR,
method='run_instance',
reason=ex)
rpc.get_notifier('scheduler').error(context, 'scheduler.run_instance',
payload)
def test_prep_resize_exception_host_in_error_state_and_raise(self):
fake_instance_uuid = 'fake-instance-id'
fake_instance = {'uuid': fake_instance_uuid}
self._mox_schedule_method_helper('select_destinations')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {
'instance_properties': {'uuid': fake_instance_uuid},
'instance_uuids': [fake_instance_uuid]
}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': fake_instance,
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.select_destinations(
self.context, request_spec, 'fake_props').AndRaise(
test.TestingException('something happened'))
inst = {
"vm_state": "",
"task_state": "",
}
old_ref, new_ref = db.instance_update_and_get_original(self.context,
fake_instance_uuid,
{"vm_state": vm_states.ERROR,
"task_state": None}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
mox.IsA(conductor_api.LocalAPI), new_ref,
mox.IsA(test.TestingException), mox.IgnoreArg())
self.mox.ReplayAll()
self.assertRaises(test.TestingException, self.manager.prep_resize,
**kwargs)
def set_vm_state_and_notify(context, service, method, updates, ex,
request_spec, db):
"""changes VM state and notifies."""
LOG.warning(_("Failed to %(service)s_%(method)s: %(ex)s"),
{'service': service, 'method': method, 'ex': ex})
vm_state = updates['vm_state']
properties = request_spec.get('instance_properties', {})
# NOTE(vish): We shouldn't get here unless we have a catastrophic
# failure, so just set all instances to error. if uuid
# is not set, instance_uuids will be set to [None], this
# is solely to preserve existing behavior and can
# be removed along with the 'if instance_uuid:' if we can
# verify that uuid is always set.
uuids = [properties.get('uuid')]
from nova.conductor import api as conductor_api
conductor = conductor_api.LocalAPI()
notifier = notify.get_notifier(service)
for instance_uuid in request_spec.get('instance_uuids') or uuids:
if instance_uuid:
state = vm_state.upper()
LOG.warning(_('Setting instance to %s state.'), state,
instance_uuid=instance_uuid)
# update instance state and notify on the transition
(old_ref, new_ref) = db.instance_update_and_get_original(
context, instance_uuid, updates)
notifications.send_update(context, old_ref, new_ref,
service=service)
compute_utils.add_instance_fault_from_exc(context,
conductor,
new_ref, ex, sys.exc_info())
payload = dict(request_spec=request_spec,
instance_properties=properties,
instance_id=instance_uuid,
state=vm_state,
method=method,
reason=ex)
event_type = '%s.%s' % (service, method)
notifier.error(context, event_type, payload)
def test_prep_resize_no_valid_host_back_in_shutoff_state(self):
fake_instance_uuid = 'fake-instance-id'
fake_instance = {'uuid': fake_instance_uuid, "vm_state": "stopped"}
inst = {"vm_state": "stopped", "task_state": ""}
self._mox_schedule_method_helper('select_destinations')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {
'instance_type': 'fake_type',
'instance_uuids': [fake_instance_uuid],
'instance_properties': {
'uuid': fake_instance_uuid
}
}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': fake_instance,
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.select_destinations(
self.context, request_spec,
'fake_props').AndRaise(exception.NoValidHost(reason=""))
old_ref, new_ref = db.instance_update_and_get_original(
self.context, fake_instance_uuid, {
"vm_state": vm_states.STOPPED,
"task_state": None
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, mox.IsA(conductor_api.LocalAPI), new_ref,
mox.IsA(exception.NoValidHost), mox.IgnoreArg())
self.mox.ReplayAll()
self.manager.prep_resize(**kwargs)
def test_upload_image_retries_then_raises_exception(self):
self.flags(num_retries=2, group='glance')
params = self._get_upload_params()
self.mox.StubOutWithMock(self.session, 'call_plugin_serialized')
self.mox.StubOutWithMock(time, 'sleep')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
error_details = ["", "", "RetryableError", ""]
error = self.session.XenAPI.Failure(details=error_details)
self.session.call_plugin_serialized('glance', 'upload_vhd2',
**params).AndRaise(error)
compute_utils.add_instance_fault_from_exc(
self.context, self.instance, error,
(fake.Failure, error, mox.IgnoreArg()))
time.sleep(0.5)
self.session.call_plugin_serialized('glance', 'upload_vhd2',
**params).AndRaise(error)
compute_utils.add_instance_fault_from_exc(
self.context, self.instance, error,
(fake.Failure, error, mox.IgnoreArg()))
time.sleep(1)
self.session.call_plugin_serialized('glance', 'upload_vhd2',
**params).AndRaise(error)
compute_utils.add_instance_fault_from_exc(
self.context, self.instance, error,
(fake.Failure, error, mox.IgnoreArg()))
self.mox.ReplayAll()
self.assertRaises(exception.CouldNotUploadImage,
self.store.upload_image, self.context, self.session,
self.instance, 'fake_image_uuid', ['fake_vdi_uuid'])
self.mox.VerifyAll()
def test_run_instance_non_admin(self):
self.was_admin = False
def fake_get(context, *args, **kwargs):
# make sure this is called with admin context, even though
# we're using user context below
self.was_admin = context.is_admin
return {}
sched = fakes.FakeFilterScheduler()
self.stubs.Set(sched.host_manager, 'get_all_host_states', fake_get)
fake_context = context.RequestContext('user', 'project')
uuid = 'fake-uuid1'
instance_properties = {'project_id': 1, 'os_type': 'Linux'}
request_spec = {
'instance_type': {
'memory_mb': 1,
'local_gb': 1
},
'instance_properties': instance_properties,
'instance_uuids': [uuid]
}
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
old_ref, new_ref = db.instance_update_and_get_original(
fake_context, uuid, {
'vm_state': vm_states.ERROR,
'task_state': None
}).AndReturn(({}, {}))
compute_utils.add_instance_fault_from_exc(
fake_context, new_ref, mox.IsA(exception.NoValidHost),
mox.IgnoreArg())
self.mox.ReplayAll()
sched.schedule_run_instance(fake_context, request_spec, None, None,
None, None, {}, False)
self.assertTrue(self.was_admin)
def test_prep_resize_no_valid_host_back_in_active_state(self):
fake_instance_uuid = 'fake-instance-id'
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper('schedule_prep_resize')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {
'instance_type': 'fake_type',
'instance_uuids': [fake_instance_uuid],
'instance_properties': {
'uuid': fake_instance_uuid
}
}
kwargs = {
'context': self.context,
'image': 'fake_image',
'request_spec': request_spec,
'filter_properties': 'fake_props',
'instance': 'fake_instance',
'instance_type': 'fake_type',
'reservations': list('fake_res'),
}
self.manager.driver.schedule_prep_resize(**kwargs).AndRaise(
exception.NoValidHost(reason=""))
db.instance_update_and_get_original(self.context, fake_instance_uuid, {
"vm_state": vm_states.ACTIVE,
"task_state": None
}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(
self.context, fake_instance_uuid, mox.IsA(exception.NoValidHost),
mox.IgnoreArg())
self.mox.ReplayAll()
self.manager.prep_resize(**kwargs)
def test_run_instance_no_hosts(self):
def _fake_empty_call_zone_method(*args, **kwargs):
return []
sched = fakes.FakeFilterScheduler()
uuid = 'fake-uuid1'
fake_context = context.RequestContext('user', 'project')
instance_properties = {'project_id': 1, 'os_type': 'Linux'}
request_spec = {
'instance_type': {
'memory_mb': 1,
'root_gb': 1,
'ephemeral_gb': 0
},
'instance_properties': instance_properties,
'instance_uuids': [uuid]
}
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
old_ref, new_ref = db.instance_update_and_get_original(
fake_context, uuid, {
'vm_state': vm_states.ERROR,
'task_state': None
}).AndReturn(({}, {}))
compute_utils.add_instance_fault_from_exc(
fake_context, new_ref, mox.IsA(exception.NoValidHost),
mox.IgnoreArg())
self.mox.StubOutWithMock(db, 'compute_node_get_all')
db.compute_node_get_all(mox.IgnoreArg()).AndReturn([])
self.mox.ReplayAll()
sched.schedule_run_instance(fake_context, request_spec, None, None,
None, None, {}, False)
def test_run_instance_exception_puts_instance_in_error_state(self):
fake_instance_uuid = 'fake-instance-id'
inst = {"vm_state": "", "task_state": ""}
self._mox_schedule_method_helper('schedule_run_instance')
self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
request_spec = {'instance_properties':
{'uuid': fake_instance_uuid}}
self.manager.driver.schedule_run_instance(self.context,
request_spec, None, None, None, None, {}).AndRaise(
exception.NoValidHost(reason=""))
db.instance_update_and_get_original(self.context, fake_instance_uuid,
{"vm_state": vm_states.ERROR,
"task_state": None}).AndReturn((inst, inst))
compute_utils.add_instance_fault_from_exc(self.context,
fake_instance_uuid, mox.IsA(exception.NoValidHost),
mox.IgnoreArg())
self.mox.ReplayAll()
self.manager.run_instance(self.context, request_spec,
None, None, None, None, {})
def _create_instance_data(self):
"""Creates an instance record and associated data like BDMs, VIFs,
migrations, etc in the source cell and returns the Instance object.
The idea is to create as many things from the
Instance.INSTANCE_OPTIONAL_ATTRS list as possible.
:returns: The created Instance and Migration objects
"""
# Create the nova-compute services record first.
fake_service = test_service._fake_service()
fake_service.pop('version', None) # version field is immutable
fake_service.pop('id', None) # cannot create with an id set
service = objects.Service(self.source_context, **fake_service)
service.create()
# Create the compute node using the service.
fake_compute_node = copy.copy(test_compute_node.fake_compute_node)
fake_compute_node['host'] = service.host
fake_compute_node['hypervisor_hostname'] = service.host
fake_compute_node['stats'] = {} # the object requires a dict
fake_compute_node['service_id'] = service.id
fake_compute_node.pop('id', None) # cannot create with an id set
compute_node = objects.ComputeNode(self.source_context,
**fake_compute_node)
compute_node.create()
# Build an Instance object with basic fields set.
updates = {
'metadata': {
'foo': 'bar'
},
'system_metadata': {
'roles': ['member']
},
'host': compute_node.host,
'node': compute_node.hypervisor_hostname
}
inst = fake_instance.fake_instance_obj(self.source_context, **updates)
delattr(inst, 'id') # cannot create an instance with an id set
# Now we have to dirty all of the fields because fake_instance_obj
# uses Instance._from_db_object to create the Instance object we have
# but _from_db_object calls obj_reset_changes() which resets all of
# the fields that were on the object, including the basic stuff like
# the 'host' field, which means those fields don't get set in the DB.
# TODO(mriedem): This should live in fake_instance_obj with a
# make_creatable kwarg.
for field in inst.obj_fields:
if field in inst:
setattr(inst, field, getattr(inst, field))
# Make sure at least one expected basic field is dirty on the Instance.
self.assertIn('host', inst.obj_what_changed())
# Set the optional fields on the instance before creating it.
inst.pci_requests = objects.InstancePCIRequests(requests=[
objects.InstancePCIRequest(
**test_instance_pci_requests.fake_pci_requests[0])
])
inst.numa_topology = objects.InstanceNUMATopology(
cells=test_instance_numa.fake_obj_numa_topology.cells)
inst.trusted_certs = objects.TrustedCerts(ids=[uuids.cert])
inst.vcpu_model = test_vcpu_model.fake_vcpumodel
inst.keypairs = objects.KeyPairList(
objects=[objects.KeyPair(**test_keypair.fake_keypair)])
inst.device_metadata = (
test_instance_device_metadata.get_fake_obj_device_metadata(
self.source_context))
# FIXME(mriedem): db.instance_create does not handle tags
inst.obj_reset_changes(['tags'])
inst.create()
bdm = {
'instance_uuid': inst.uuid,
'source_type': 'volume',
'destination_type': 'volume',
'volume_id': uuids.volume_id,
'volume_size': 1,
'device_name': '/dev/vda',
}
bdm = objects.BlockDeviceMapping(
self.source_context,
**fake_block_device.FakeDbBlockDeviceDict(bdm_dict=bdm))
delattr(bdm, 'id') # cannot create a bdm with an id set
bdm.obj_reset_changes(['id'])
bdm.create()
vif = objects.VirtualInterface(self.source_context,
address='de:ad:be:ef:ca:fe',
uuid=uuids.port,
instance_uuid=inst.uuid)
vif.create()
info_cache = objects.InstanceInfoCache().new(self.source_context,
inst.uuid)
info_cache.network_info = network_model.NetworkInfo(
[network_model.VIF(id=vif.uuid, address=vif.address)])
info_cache.save(update_cells=False)
objects.TagList.create(self.source_context, inst.uuid, ['test'])
try:
raise test.TestingException('test-fault')
except test.TestingException as fault:
compute_utils.add_instance_fault_from_exc(self.source_context,
inst, fault)
objects.InstanceAction().action_start(self.source_context,
inst.uuid,
'resize',
want_result=False)
objects.InstanceActionEvent().event_start(self.source_context,
inst.uuid,
'migrate_server',
want_result=False)
# Create a fake migration for the cross-cell resize operation.
migration = objects.Migration(
self.source_context,
**test_migration.fake_db_migration(instance_uuid=inst.uuid,
cross_cell_move=True,
migration_type='resize'))
delattr(migration, 'id') # cannot create a migration with an id set
migration.obj_reset_changes(['id'])
migration.create()
# Create an old non-resize migration to make sure it is copied to the
# target cell database properly.
old_migration = objects.Migration(
self.source_context,
**test_migration.fake_db_migration(instance_uuid=inst.uuid,
migration_type='live-migration',
status='completed',
uuid=uuids.old_migration))
delattr(old_migration, 'id') # cannot create a migration with an id
old_migration.obj_reset_changes(['id'])
old_migration.create()
fake_pci_device = copy.copy(test_pci_device.fake_db_dev)
fake_pci_device['extra_info'] = {} # the object requires a dict
fake_pci_device['compute_node_id'] = compute_node.id
pci_device = objects.PciDevice.create(self.source_context,
fake_pci_device)
pci_device.allocate(inst) # sets the status and instance_uuid fields
pci_device.save()
# Return a fresh copy of the instance from the DB with as many joined
# fields loaded as possible.
expected_attrs = copy.copy(instance_obj.INSTANCE_OPTIONAL_ATTRS)
# Cannot load fault from get_by_uuid.
expected_attrs.remove('fault')
inst = objects.Instance.get_by_uuid(self.source_context,
inst.uuid,
expected_attrs=expected_attrs)
return inst, migration
def retry_cb(context, instance, exc=None):
if exc:
exc_info = sys.exc_info()
LOG.debug(exc.message, exc_info=exc_info)
compute_utils.add_instance_fault_from_exc(
context, instance, exc, exc_info)