def _recoverExistingVms(self):
start_time = vdsm.common.time.monotonic_time()
try:
self.log.debug('recovery: started')
# Starting up libvirt might take long when host under high load,
# we prefer running this code in external thread to avoid blocking
# API response.
mog = min(config.getint('vars', 'max_outgoing_migrations'),
numa.cpu_topology().cores)
migration.SourceThread.ongoingMigrations.bound = mog
recovery.all_domains(self)
# recover stage 3: waiting for domains to go up
self._waitForDomainsUp()
self._recovery = False
# Now if we have VMs to restore we should wait pool connection
# and then prepare all volumes.
# Actually, we need it just to get the resources for future
# volumes manipulations
self._waitForStoragePool()
self._preparePathsForRecoveredVMs()
self.log.info('recovery: completed in %is',
vdsm.common.time.monotonic_time() - start_time)
except:
self.log.exception("recovery: failed")
raise
def _recoverExistingVms(self):
start_time = vdsm.common.time.monotonic_time()
try:
self.log.debug('recovery: started')
# Starting up libvirt might take long when host under high load,
# we prefer running this code in external thread to avoid blocking
# API response.
mog = min(config.getint('vars', 'max_outgoing_migrations'),
numa.cpu_topology().cores)
migration.SourceThread.ongoingMigrations.bound = mog
recovery.all_domains(self)
# recover stage 3: waiting for domains to go up
self._waitForDomainsUp()
recovery.clean_vm_files(self)
self._recovery = False
# Now if we have VMs to restore we should wait pool connection
# and then prepare all volumes.
# Actually, we need it just to get the resources for future
# volumes manipulations
self._waitForStoragePool()
self._preparePathsForRecoveredVMs()
self.log.info('recovery: completed in %is',
vdsm.common.time.monotonic_time() - start_time)
except:
self.log.exception("recovery: failed")
raise
def test_domain_error(self):
"""
We find VMs to recover through libvirt, but we get a failure trying
to identify (UUIDString, XMLDesc) a domain being recovered
"""
self.conn.domains['a'].XMLDesc = _raise
recovery.all_domains(self.cif)
self.assertEqual(set(self.cif.vmRequests.keys()), set(('b', )))
def test_recover_failures(self, create_fn):
"""
We find VMs to recover through libvirt, but Vdsm fail to create
its Vm objects. We should then destroy those VMs.
"""
with MonkeyPatchScope([(self.cif, 'createVm', create_fn)]):
recovery.all_domains(self.cif)
self.assertEqual(self.cif.vmRequests, {})
self.assertTrue(all(vm.destroyed for vm in self.conn.domains.values()))
def test_recover_external_vm_down(self):
vm_is_ext = [True] * len(self.vm_uuids)
self.conn.domains = _make_domains_collection(
list(zip(self.vm_uuids, vm_is_ext)))
for dom in self.conn.domains.values():
dom.domState = libvirt.VIR_DOMAIN_SHUTOFF
recovery.all_domains(self.cif)
assert self.cif.vmRequests == {}
def test_recover_external_vm_down(self):
vm_is_ext = [True] * len(self.vm_uuids)
self.conn.domains = _make_domains_collection(
list(zip(self.vm_uuids, vm_is_ext))
)
for dom in self.conn.domains.values():
dom.domState = libvirt.VIR_DOMAIN_SHUTOFF
recovery.all_domains(self.cif)
self.assertEqual(self.cif.vmRequests, {})
def test_recover_few_domains(self):
recovery.all_domains(self.cif)
self.assertEqual(
set(self.cif.vmRequests.keys()),
set(self.vm_uuids)
)
self.assertEqual(
self.vm_is_ext,
[conf['external'] for conf, _ in self.cif.vmRequests.values()]
)
def test_domain_error(self):
"""
We find VMs to recover through libvirt, but we get a failure trying
to identify (UUIDString, XMLDesc) a domain being recovered
"""
self.conn.domains['a'].XMLDesc = _raise
recovery.all_domains(self.cif)
self.assertEqual(
set(self.cif.vmRequests.keys()),
set(('b',))
)
def test_recover_external_vm_down(self):
vm_uuids = ('a', 'b',)
vm_is_ext = [True] * len(vm_uuids)
self.conn.domains = _make_domains_collection(
zip(vm_uuids, vm_is_ext)
)
for dom in self.conn.domains.values():
dom.domState = libvirt.VIR_DOMAIN_SHUTOFF
recovery.all_domains(self.cif)
self.assertEqual(self.cif.vmRequests, {})
def test_without_any_vms(self):
with namedTemporaryDir() as tmpdir:
with MonkeyPatchScope([
(constants, 'P_VDSM_RUN', tmpdir),
(recovery, '_list_domains', lambda: []),
(containersconnection, 'recovery', lambda: []),
]):
fakecif = fake.ClientIF()
recovery.all_domains(fakecif)
self.assertEqual(fakecif.vmContainer, {})
def test_without_any_vms(self):
with namedTemporaryDir() as tmpdir:
with MonkeyPatchScope([
(constants, 'P_VDSM_RUN', tmpdir),
(recovery, '_list_domains', lambda: []),
(containersconnection, 'recovery', lambda: []),
]):
fakecif = fake.ClientIF()
recovery.all_domains(fakecif)
self.assertEqual(fakecif.vmContainer, {})
def test_recover_external_vm_error(self):
"""
handle gracefully error while getting the state of external VM
"""
vm_is_ext = [True] * len(self.vm_uuids)
self.conn.domains = _make_domains_collection(
list(zip(self.vm_uuids, vm_is_ext)))
for dom in self.conn.domains.values():
dom.state = _raise
recovery.all_domains(self.cif)
assert self.cif.vmRequests == {}
def test_external_vm(self):
vm_infos = (('a', True), ('b', False),)
self.conn.domains = _make_domains_collection(vm_infos)
recovery.all_domains(self.cif)
self.assertEqual(
set(self.cif.vmRequests.keys()),
set(vm_id for vm_id, _ in vm_infos)
)
for vm_id, vm_is_ext in vm_infos:
conf, _ = self.cif.vmRequests[vm_id]
self.assertEqual(vm_is_ext, conf['external'])
def test_recover_and_destroy_failure(self):
"""
We find VMs to recover through libvirt, but Vdsm fail to create
its Vm objects. We should then destroy those VMs, but one of
the domains can't complete that. We should handle this case
gracefully
"""
self.conn.domains['b'].destroy = _raise
with MonkeyPatchScope([(self.cif, 'createVm', _error)]):
recovery.all_domains(self.cif)
self.assertEqual(self.cif.vmRequests, {})
self.assertTrue(self.conn.domains['a'].destroyed)
self.assertFalse(self.conn.domains['b'].destroyed)
def test_recover_external_vm_error(self):
"""
handle gracefully error while getting the state of external VM
"""
vm_is_ext = [True] * len(self.vm_uuids)
self.conn.domains = _make_domains_collection(
list(zip(self.vm_uuids, vm_is_ext))
)
for dom in self.conn.domains.values():
dom.state = _raise
recovery.all_domains(self.cif)
self.assertEqual(self.cif.vmRequests, {})
def test_recover_few_domains(self):
vm_uuids = ('a', 'b',)
vm_is_ext = [False] * len(vm_uuids)
self.conn.domains = _make_domains_collection(
zip(vm_uuids, vm_is_ext)
)
recovery.all_domains(self.cif)
self.assertEqual(
set(self.cif.vmRequests.keys()),
set(vm_uuids)
)
self.assertEqual(
vm_is_ext,
[conf['external'] for conf, _ in self.cif.vmRequests.values()]
)
def test_recover_failures(self, create_fn):
"""
We find VMs to recover through libvirt, but Vdsm fail to create
its Vm objects. We should then destroy those VMs.
"""
with MonkeyPatchScope([
(self.cif, 'createVm', create_fn)
]):
recovery.all_domains(self.cif)
self.assertEqual(
self.cif.vmRequests,
{}
)
self.assertTrue(all(
vm.destroyed for vm in self.conn.domains.values()
))
def test_domain_error(self):
"""
We find VMs to recover through libvirt, but we get a failure trying
to identify (UUIDString, XMLDesc) a domain being recovered
"""
vm_uuids = ('a', 'b',)
vm_is_ext = [False] * len(vm_uuids)
self.conn.domains = _make_domains_collection(
zip(vm_uuids, vm_is_ext)
)
self.conn.domains['a'].XMLDesc = _raise
recovery.all_domains(self.cif)
self.assertEqual(
set(self.cif.vmRequests.keys()),
set(('b',))
)
def test_external_vm_failure(self):
"""
We find *external* VMs to recover through libvirt,
but Vdsm fail to create its Vm objects.
We should then destroy the non-external VMs.
"""
vm_infos = (
('a', True),
('b', False),
)
self.conn.domains = _make_domains_collection(vm_infos)
with MonkeyPatchScope([(self.cif, 'createVm', _error)]):
recovery.all_domains(self.cif)
self.assertEqual(self.cif.vmRequests, {})
for vm_id, vm_is_ext in vm_infos:
vm_obj = self.conn.domains[vm_id]
expect_destroy = not vm_is_ext
self.assertEqual(vm_obj.destroyed, expect_destroy)
def test_recover_and_destroy_failure(self):
"""
We find VMs to recover through libvirt, but Vdsm fail to create
its Vm objects. We should then destroy those VMs, but one of
the domains can't complete that. We should handle this case
gracefully
"""
self.conn.domains['b'].destroy = _raise
with MonkeyPatchScope([
(self.cif, 'createVm', _error)
]):
recovery.all_domains(self.cif)
self.assertEqual(
self.cif.vmRequests,
{}
)
self.assertTrue(self.conn.domains['a'].destroyed)
self.assertFalse(self.conn.domains['b'].destroyed)
def test_external_vm_failure(self):
"""
We find *external* VMs to recover through libvirt,
but Vdsm fail to create its Vm objects.
We should then destroy the non-external VMs.
"""
vm_infos = (('a', True), ('b', False),)
self.conn.domains = _make_domains_collection(vm_infos)
with MonkeyPatchScope([
(self.cif, 'createVm', _error)
]):
recovery.all_domains(self.cif)
self.assertEqual(
self.cif.vmRequests,
{}
)
for vm_id, vm_is_ext in vm_infos:
vm_obj = self.conn.domains[vm_id]
expect_destroy = not vm_is_ext
self.assertEqual(vm_obj.destroyed, expect_destroy)
def test_recover_failures(self, create_fn):
"""
We find VMs to recover through libvirt, but Vdsm fail to create
its Vm objects. We should then destroy those VMs.
"""
vm_uuids = ('a', 'b',)
vm_is_ext = [False] * len(vm_uuids)
self.conn.domains = _make_domains_collection(
zip(vm_uuids, vm_is_ext)
)
with MonkeyPatchScope([
(self.cif, 'createVm', create_fn)
]):
recovery.all_domains(self.cif)
self.assertEqual(
self.cif.vmRequests,
{}
)
self.assertTrue(all(
vm.destroyed for vm in self.conn.domains.values()
))
def test_recover_few_domains(self):
recovery.all_domains(self.cif)
assert set(self.cif.vmRequests.keys()) == \
set(self.vm_uuids)
assert self.vm_is_ext == \
[conf['external'] for conf, _ in self.cif.vmRequests.values()]
def test_recover_no_domains(self):
self.conn.domains = {}
recovery.all_domains(self.cif)
assert self.cif.vmRequests == {}
def test_recover_no_domains(self):
self.conn.domains = {}
recovery.all_domains(self.cif)
self.assertEqual(self.cif.vmRequests, {})
def test_recover_no_domains(self):
self.conn.domains = {}
recovery.all_domains(self.cif)
self.assertEqual(self.cif.vmRequests, {})
