def test_upload_image_retries_on_signal_exception(self): self.flags(num_retries=2, group='glance') params = self._get_upload_params() self.mox.StubOutWithMock(self.session, 'call_plugin_serialized') self.mox.StubOutWithMock(time, 'sleep') self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc') error_details = ["", "task signaled", "", ""] error = self.session.XenAPI.Failure(details=error_details) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc(self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) time.sleep(0.5) # Note(johngarbutt) XenServer 6.1 and later has this error error_details = ["", "signal: SIGTERM", "", ""] error = self.session.XenAPI.Failure(details=error_details) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc(self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) time.sleep(1) self.session.call_plugin_serialized('glance', 'upload_vhd', **params) self.mox.ReplayAll() self.store.upload_image(self.context, self.session, self.instance, 'fake_image_uuid', ['fake_vdi_uuid']) self.mox.VerifyAll()
def test_upload_image_retries_on_signal_exception(self): self.flags(num_retries=2, group='glance') params = self._get_upload_params() self.mox.StubOutWithMock(self.session, 'call_plugin_serialized') self.mox.StubOutWithMock(time, 'sleep') self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc') error_details = ["", "task signaled", "", ""] error = self.session.XenAPI.Failure(details=error_details) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc( self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) time.sleep(0.5) # Note(johngarbutt) XenServer 6.1 and later has this error error_details = ["", "signal: SIGTERM", "", ""] error = self.session.XenAPI.Failure(details=error_details) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc( self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) time.sleep(1) self.session.call_plugin_serialized('glance', 'upload_vhd', **params) self.mox.ReplayAll() self.store.upload_image(self.context, self.session, self.instance, 'fake_image_uuid', ['fake_vdi_uuid']) self.mox.VerifyAll()
def _add_instance_fault(self, error, exc_info): LOG.warning(_LW("Ignoring error while configuring instance with " "agent: %s"), error, instance=self.instance, exc_info=True) try: ctxt = context.get_admin_context() compute_utils.add_instance_fault_from_exc( ctxt, self.instance, error, exc_info=exc_info) except Exception: LOG.debug("Error setting instance fault.", exc_info=True)
def test_set_vm_state_and_notify(self): expected_uuid = 'fake-uuid' request_spec = dict(instance_properties=dict(uuid='other-uuid')) updates = dict(vm_state='fake-vm-state') service = 'fake-service' method = 'fake-method' exc_info = 'exc_info' self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc') self.mox.StubOutWithMock(notifications, 'send_update') self.mox.StubOutWithMock(db, 'instance_update_and_get_original') self.mox.StubOutWithMock(rpc, 'get_notifier') notifier = self.mox.CreateMockAnything() rpc.get_notifier(service).AndReturn(notifier) old_ref = 'old_ref' new_ref = 'new_ref' inst_obj = 'inst_obj' db.instance_update_and_get_original( self.context, expected_uuid, updates, columns_to_join=['system_metadata']).AndReturn((old_ref, new_ref)) notifications.send_update(self.context, old_ref, inst_obj, service=service) compute_utils.add_instance_fault_from_exc( self.context, inst_obj, exc_info, mox.IsA(tuple)) payload = dict(request_spec=request_spec, instance_properties=request_spec.get( 'instance_properties', {}), instance_id=expected_uuid, state='fake-vm-state', method=method, reason=exc_info) event_type = '%s.%s' % (service, method) notifier.error(self.context, event_type, payload) self.mox.ReplayAll() with mock.patch.object(objects.Instance, '_from_db_object', return_value=inst_obj): scheduler_utils.set_vm_state_and_notify(self.context, expected_uuid, service, method, updates, exc_info, request_spec, db)
def test_set_vm_state_and_notify(self): expected_uuid = 'fake-uuid' request_spec = dict(instance_properties=dict(uuid='other-uuid')) updates = dict(vm_state='fake-vm-state') service = 'fake-service' method = 'fake-method' exc_info = 'exc_info' self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc') self.mox.StubOutWithMock(notifications, 'send_update') self.mox.StubOutWithMock(db, 'instance_update_and_get_original') self.mox.StubOutWithMock(rpc, 'get_notifier') notifier = self.mox.CreateMockAnything() rpc.get_notifier(service).AndReturn(notifier) old_ref = 'old_ref' new_ref = 'new_ref' inst_obj = 'inst_obj' db.instance_update_and_get_original( self.context, expected_uuid, updates, columns_to_join=['system_metadata']).AndReturn((old_ref, new_ref)) notifications.send_update(self.context, old_ref, inst_obj, service=service) compute_utils.add_instance_fault_from_exc(self.context, inst_obj, exc_info, mox.IsA(tuple)) payload = dict(request_spec=request_spec, instance_properties=request_spec.get( 'instance_properties', {}), instance_id=expected_uuid, state='fake-vm-state', method=method, reason=exc_info) event_type = '%s.%s' % (service, method) notifier.error(self.context, event_type, payload) self.mox.ReplayAll() with mock.patch.object(objects.Instance, '_from_db_object', return_value=inst_obj): scheduler_utils.set_vm_state_and_notify(self.context, expected_uuid, service, method, updates, exc_info, request_spec, db)
def set_vm_state_and_notify(context, instance_uuid, service, method, updates, ex, request_spec, db): """changes VM state and notifies.""" LOG.warning(_LW("Failed to %(service)s_%(method)s: %(ex)s"), { 'service': service, 'method': method, 'ex': ex }) vm_state = updates['vm_state'] properties = request_spec.get('instance_properties', {}) # NOTE(vish): We shouldn't get here unless we have a catastrophic # failure, so just set the instance to its internal state notifier = rpc.get_notifier(service) state = vm_state.upper() LOG.warning(_LW('Setting instance to %s state.'), state, instance_uuid=instance_uuid) # update instance state and notify on the transition # NOTE(hanlind): the send_update() call below is going to want to # know about the flavor, so we need to join the appropriate things # here and objectify the results. (old_ref, new_ref) = db.instance_update_and_get_original( context, instance_uuid, updates, columns_to_join=['system_metadata']) inst_obj = objects.Instance._from_db_object( context, objects.Instance(), new_ref, expected_attrs=['system_metadata']) notifications.send_update(context, old_ref, inst_obj, service=service) compute_utils.add_instance_fault_from_exc(context, inst_obj, ex, sys.exc_info()) payload = dict(request_spec=request_spec, instance_properties=properties, instance_id=instance_uuid, state=vm_state, method=method, reason=ex) event_type = '%s.%s' % (service, method) notifier.error(context, event_type, payload)
def test_upload_image_retries_then_raises_exception(self): self.flags(num_retries=2, group='glance') params = self._get_upload_params() self.mox.StubOutWithMock(self.session, 'call_plugin_serialized') self.mox.StubOutWithMock(time, 'sleep') self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc') error_details = ["", "", "RetryableError", ""] error = self.session.XenAPI.Failure(details=error_details) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc( self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) time.sleep(0.5) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc( self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) time.sleep(1) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc( self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) self.mox.ReplayAll() self.assertRaises(exception.CouldNotUploadImage, self.store.upload_image, self.context, self.session, self.instance, 'fake_image_uuid', ['fake_vdi_uuid']) self.mox.VerifyAll()
def test_upload_image_retries_then_raises_exception(self): self.flags(num_retries=2, group='glance') params = self._get_upload_params() self.mox.StubOutWithMock(self.session, 'call_plugin_serialized') self.mox.StubOutWithMock(time, 'sleep') self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc') error_details = ["", "", "RetryableError", ""] error = self.session.XenAPI.Failure(details=error_details) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc(self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) time.sleep(0.5) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc(self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) time.sleep(1) self.session.call_plugin_serialized('glance', 'upload_vhd', **params).AndRaise(error) compute_utils.add_instance_fault_from_exc(self.context, self.instance, error, (fake.Failure, error, mox.IgnoreArg())) self.mox.ReplayAll() self.assertRaises(exception.CouldNotUploadImage, self.store.upload_image, self.context, self.session, self.instance, 'fake_image_uuid', ['fake_vdi_uuid']) self.mox.VerifyAll()
def retry_cb(context, instance, exc=None): if exc: exc_info = sys.exc_info() LOG.debug(exc.message, exc_info=exc_info) compute_utils.add_instance_fault_from_exc( context, instance, exc, exc_info)