def _recoverExistingVms(self): start_time = vdsm.common.time.monotonic_time() try: self.log.debug('recovery: started') # Starting up libvirt might take long when host under high load, # we prefer running this code in external thread to avoid blocking # API response. mog = min(config.getint('vars', 'max_outgoing_migrations'), numa.cpu_topology().cores) migration.SourceThread.ongoingMigrations.bound = mog recovery.all_domains(self) # recover stage 3: waiting for domains to go up self._waitForDomainsUp() self._recovery = False # Now if we have VMs to restore we should wait pool connection # and then prepare all volumes. # Actually, we need it just to get the resources for future # volumes manipulations self._waitForStoragePool() self._preparePathsForRecoveredVMs() self.log.info('recovery: completed in %is', vdsm.common.time.monotonic_time() - start_time) except: self.log.exception("recovery: failed") raise
def _recoverExistingVms(self): start_time = vdsm.common.time.monotonic_time() try: self.log.debug('recovery: started') # Starting up libvirt might take long when host under high load, # we prefer running this code in external thread to avoid blocking # API response. mog = min(config.getint('vars', 'max_outgoing_migrations'), numa.cpu_topology().cores) migration.SourceThread.ongoingMigrations.bound = mog recovery.all_domains(self) # recover stage 3: waiting for domains to go up self._waitForDomainsUp() recovery.clean_vm_files(self) self._recovery = False # Now if we have VMs to restore we should wait pool connection # and then prepare all volumes. # Actually, we need it just to get the resources for future # volumes manipulations self._waitForStoragePool() self._preparePathsForRecoveredVMs() self.log.info('recovery: completed in %is', vdsm.common.time.monotonic_time() - start_time) except: self.log.exception("recovery: failed") raise
def test_domain_error(self): """ We find VMs to recover through libvirt, but we get a failure trying to identify (UUIDString, XMLDesc) a domain being recovered """ self.conn.domains['a'].XMLDesc = _raise recovery.all_domains(self.cif) self.assertEqual(set(self.cif.vmRequests.keys()), set(('b', )))
def test_recover_failures(self, create_fn): """ We find VMs to recover through libvirt, but Vdsm fail to create its Vm objects. We should then destroy those VMs. """ with MonkeyPatchScope([(self.cif, 'createVm', create_fn)]): recovery.all_domains(self.cif) self.assertEqual(self.cif.vmRequests, {}) self.assertTrue(all(vm.destroyed for vm in self.conn.domains.values()))
def test_recover_external_vm_down(self): vm_is_ext = [True] * len(self.vm_uuids) self.conn.domains = _make_domains_collection( list(zip(self.vm_uuids, vm_is_ext))) for dom in self.conn.domains.values(): dom.domState = libvirt.VIR_DOMAIN_SHUTOFF recovery.all_domains(self.cif) assert self.cif.vmRequests == {}
def test_recover_external_vm_down(self): vm_is_ext = [True] * len(self.vm_uuids) self.conn.domains = _make_domains_collection( list(zip(self.vm_uuids, vm_is_ext)) ) for dom in self.conn.domains.values(): dom.domState = libvirt.VIR_DOMAIN_SHUTOFF recovery.all_domains(self.cif) self.assertEqual(self.cif.vmRequests, {})
def test_recover_few_domains(self): recovery.all_domains(self.cif) self.assertEqual( set(self.cif.vmRequests.keys()), set(self.vm_uuids) ) self.assertEqual( self.vm_is_ext, [conf['external'] for conf, _ in self.cif.vmRequests.values()] )
def test_domain_error(self): """ We find VMs to recover through libvirt, but we get a failure trying to identify (UUIDString, XMLDesc) a domain being recovered """ self.conn.domains['a'].XMLDesc = _raise recovery.all_domains(self.cif) self.assertEqual( set(self.cif.vmRequests.keys()), set(('b',)) )
def test_recover_external_vm_down(self): vm_uuids = ('a', 'b',) vm_is_ext = [True] * len(vm_uuids) self.conn.domains = _make_domains_collection( zip(vm_uuids, vm_is_ext) ) for dom in self.conn.domains.values(): dom.domState = libvirt.VIR_DOMAIN_SHUTOFF recovery.all_domains(self.cif) self.assertEqual(self.cif.vmRequests, {})
def test_without_any_vms(self): with namedTemporaryDir() as tmpdir: with MonkeyPatchScope([ (constants, 'P_VDSM_RUN', tmpdir), (recovery, '_list_domains', lambda: []), (containersconnection, 'recovery', lambda: []), ]): fakecif = fake.ClientIF() recovery.all_domains(fakecif) self.assertEqual(fakecif.vmContainer, {})
def test_recover_external_vm_error(self): """ handle gracefully error while getting the state of external VM """ vm_is_ext = [True] * len(self.vm_uuids) self.conn.domains = _make_domains_collection( list(zip(self.vm_uuids, vm_is_ext))) for dom in self.conn.domains.values(): dom.state = _raise recovery.all_domains(self.cif) assert self.cif.vmRequests == {}
def test_external_vm(self): vm_infos = (('a', True), ('b', False),) self.conn.domains = _make_domains_collection(vm_infos) recovery.all_domains(self.cif) self.assertEqual( set(self.cif.vmRequests.keys()), set(vm_id for vm_id, _ in vm_infos) ) for vm_id, vm_is_ext in vm_infos: conf, _ = self.cif.vmRequests[vm_id] self.assertEqual(vm_is_ext, conf['external'])
def test_recover_and_destroy_failure(self): """ We find VMs to recover through libvirt, but Vdsm fail to create its Vm objects. We should then destroy those VMs, but one of the domains can't complete that. We should handle this case gracefully """ self.conn.domains['b'].destroy = _raise with MonkeyPatchScope([(self.cif, 'createVm', _error)]): recovery.all_domains(self.cif) self.assertEqual(self.cif.vmRequests, {}) self.assertTrue(self.conn.domains['a'].destroyed) self.assertFalse(self.conn.domains['b'].destroyed)
def test_recover_external_vm_error(self): """ handle gracefully error while getting the state of external VM """ vm_is_ext = [True] * len(self.vm_uuids) self.conn.domains = _make_domains_collection( list(zip(self.vm_uuids, vm_is_ext)) ) for dom in self.conn.domains.values(): dom.state = _raise recovery.all_domains(self.cif) self.assertEqual(self.cif.vmRequests, {})
def test_recover_few_domains(self): vm_uuids = ('a', 'b',) vm_is_ext = [False] * len(vm_uuids) self.conn.domains = _make_domains_collection( zip(vm_uuids, vm_is_ext) ) recovery.all_domains(self.cif) self.assertEqual( set(self.cif.vmRequests.keys()), set(vm_uuids) ) self.assertEqual( vm_is_ext, [conf['external'] for conf, _ in self.cif.vmRequests.values()] )
def test_recover_failures(self, create_fn): """ We find VMs to recover through libvirt, but Vdsm fail to create its Vm objects. We should then destroy those VMs. """ with MonkeyPatchScope([ (self.cif, 'createVm', create_fn) ]): recovery.all_domains(self.cif) self.assertEqual( self.cif.vmRequests, {} ) self.assertTrue(all( vm.destroyed for vm in self.conn.domains.values() ))
def test_domain_error(self): """ We find VMs to recover through libvirt, but we get a failure trying to identify (UUIDString, XMLDesc) a domain being recovered """ vm_uuids = ('a', 'b',) vm_is_ext = [False] * len(vm_uuids) self.conn.domains = _make_domains_collection( zip(vm_uuids, vm_is_ext) ) self.conn.domains['a'].XMLDesc = _raise recovery.all_domains(self.cif) self.assertEqual( set(self.cif.vmRequests.keys()), set(('b',)) )
def test_external_vm_failure(self): """ We find *external* VMs to recover through libvirt, but Vdsm fail to create its Vm objects. We should then destroy the non-external VMs. """ vm_infos = ( ('a', True), ('b', False), ) self.conn.domains = _make_domains_collection(vm_infos) with MonkeyPatchScope([(self.cif, 'createVm', _error)]): recovery.all_domains(self.cif) self.assertEqual(self.cif.vmRequests, {}) for vm_id, vm_is_ext in vm_infos: vm_obj = self.conn.domains[vm_id] expect_destroy = not vm_is_ext self.assertEqual(vm_obj.destroyed, expect_destroy)
def test_recover_and_destroy_failure(self): """ We find VMs to recover through libvirt, but Vdsm fail to create its Vm objects. We should then destroy those VMs, but one of the domains can't complete that. We should handle this case gracefully """ self.conn.domains['b'].destroy = _raise with MonkeyPatchScope([ (self.cif, 'createVm', _error) ]): recovery.all_domains(self.cif) self.assertEqual( self.cif.vmRequests, {} ) self.assertTrue(self.conn.domains['a'].destroyed) self.assertFalse(self.conn.domains['b'].destroyed)
def test_external_vm_failure(self): """ We find *external* VMs to recover through libvirt, but Vdsm fail to create its Vm objects. We should then destroy the non-external VMs. """ vm_infos = (('a', True), ('b', False),) self.conn.domains = _make_domains_collection(vm_infos) with MonkeyPatchScope([ (self.cif, 'createVm', _error) ]): recovery.all_domains(self.cif) self.assertEqual( self.cif.vmRequests, {} ) for vm_id, vm_is_ext in vm_infos: vm_obj = self.conn.domains[vm_id] expect_destroy = not vm_is_ext self.assertEqual(vm_obj.destroyed, expect_destroy)
def test_recover_failures(self, create_fn): """ We find VMs to recover through libvirt, but Vdsm fail to create its Vm objects. We should then destroy those VMs. """ vm_uuids = ('a', 'b',) vm_is_ext = [False] * len(vm_uuids) self.conn.domains = _make_domains_collection( zip(vm_uuids, vm_is_ext) ) with MonkeyPatchScope([ (self.cif, 'createVm', create_fn) ]): recovery.all_domains(self.cif) self.assertEqual( self.cif.vmRequests, {} ) self.assertTrue(all( vm.destroyed for vm in self.conn.domains.values() ))
def test_recover_few_domains(self): recovery.all_domains(self.cif) assert set(self.cif.vmRequests.keys()) == \ set(self.vm_uuids) assert self.vm_is_ext == \ [conf['external'] for conf, _ in self.cif.vmRequests.values()]
def test_recover_no_domains(self): self.conn.domains = {} recovery.all_domains(self.cif) assert self.cif.vmRequests == {}
def test_recover_no_domains(self): self.conn.domains = {} recovery.all_domains(self.cif) self.assertEqual(self.cif.vmRequests, {})