def validate(self): """ Validate that the storage domain is accessible. """ self.log.info("sdUUID=%s", self.sdUUID) self.invalidateMetadata() if not len(self.getMetadata()): raise se.StorageDomainAccessError(self.sdUUID)
def chkVG(vgName): cmd = ["vgck", vgName] rc, out, err = _lvminfo.cmd(cmd, _lvminfo._getVGDevs((vgName, ))) if rc != 0: _lvminfo._invalidatevgs(vgName) _lvminfo._invalidatelvs(vgName) raise se.StorageDomainAccessError("%s: %s" % (vgName, err)) return True
def getStats(self): """ Get storage domain statistics """ # self.log.info("sdUUID=%s", self.sdUUID) stats = {'disktotal': '', 'diskfree': '', 'mdavalid': True, 'mdathreshold': True, 'mdasize': 0, 'mdafree': 0} try: st = self.oop.os.statvfs(self.domaindir) stats['disktotal'] = str(st.f_frsize * st.f_blocks) stats['diskfree'] = str(st.f_frsize * st.f_bavail) except OSError as e: self.log.info("sdUUID=%s %s", self.sdUUID, str(e)) if e.errno == errno.ESTALE: raise se.FileStorageDomainStaleNFSHandle raise se.StorageDomainAccessError(self.sdUUID) return stats
class TestMonitorThreadMonitoring(VdsmTestCase): # In this state we do: # 1. If refresh timeout has expired, remove the domain from the cache # 2. call domain.selftest() # 3. call domain.getStats() # 4. call domain.validateMaster() # 5. call domain.hasHostId() # 6. call domain.getVersion() # # - When path check completes, we get a callback from the checker thread # and update monitor status. # - On failure, we abort the process, and set status.error. # - If this is the first failure, we emit a domain state change event with # valid=False. # - On success, if the previous status.error was set, we emit a domain # state change event with valid=True. # - If everything was successful, and host id not acquired yet, acquire it # - We repeat this forever until the monitor is stopped. def test_unknown_to_valid(self): with monitor_env() as env: domain = FakeDomain("uuid") monitor.sdCache.domains["uuid"] = domain env.thread.start() # First cycle suceeds, but path status is not avialale yet env.wait_for_cycle() status = env.thread.getStatus() self.assertFalse(status.actual) self.assertEqual(env.event.received, []) # When path succeeds, emit VALID event env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) status = env.thread.getStatus() self.assertTrue(status.actual) self.assertTrue(status.valid) self.assertEqual(env.event.received, [(('uuid', True), {})]) @permutations([ ("selftest", OSError), ("selftest", UnexpectedError), ("getStats", se.FileStorageDomainStaleNFSHandle), # TODO: Uncomment when test is fixed to not check a type. # ("getStats", se.StorageDomainAccessError("fake-sd-id")), ("getStats", UnexpectedError), ("validateMaster", OSError), ("validateMaster", UnexpectedError), ("getVersion", OSError), ("getVersion", UnexpectedError), ]) def test_from_unknown_to_invalid_domain(self, method, exception): with monitor_env() as env: domain = FakeDomain("uuid") domain.errors[method] = exception monitor.sdCache.domains["uuid"] = domain env.thread.start() # First cycle fail, emit event without waiting for path status env.wait_for_cycle() status = env.thread.getStatus() self.assertTrue(status.actual) self.assertFalse(status.valid) # TODO: Should assert we got an exception set by the test instead # of checking a type. self.assertIsInstance(status.error, exception) self.assertEqual(env.event.received, [(('uuid', False), {})]) @permutations([[se.MiscFileReadException], [UnexpectedError]]) def test_from_unknown_to_invalid_path(self, exception): with monitor_env() as env: domain = FakeDomain("uuid") monitor.sdCache.domains["uuid"] = domain env.thread.start() # First cycle succeed, but path status is not available yet env.wait_for_cycle() status = env.thread.getStatus() self.assertFalse(status.actual) self.assertEqual(env.event.received, []) # When path fail, emit INVALID event env.checker.complete(domain.getMonitoringPath(), FakeCheckResult(exception)) status = env.thread.getStatus() self.assertTrue(status.actual) self.assertFalse(status.valid) self.assertIsInstance(status.error, exception) self.assertEqual(env.event.received, [(('uuid', False), {})]) @permutations([ ("selftest", OSError), ("selftest", UnexpectedError), ("getStats", se.FileStorageDomainStaleNFSHandle), ("getStats", se.StorageDomainAccessError("fake-sd-id")), ("getStats", UnexpectedError), ("validateMaster", OSError), ("validateMaster", UnexpectedError), ("getVersion", OSError), ("getVersion", UnexpectedError), ]) def test_from_invalid_to_valid_domain(self, method, exception): with monitor_env() as env: domain = FakeDomain("uuid") domain.errors[method] = exception monitor.sdCache.domains["uuid"] = domain env.thread.start() # First cycle fail, and emit INVALID event env.wait_for_cycle() del env.event.received[0] # Path status succeeds, but domain status is not valid, so no event # is emitted. env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) status = env.thread.getStatus() self.assertTrue(status.actual) self.assertFalse(status.valid) self.assertEqual(env.event.received, []) # When next cycle succeeds, emit VALID event del domain.errors[method] env.wait_for_cycle() status = env.thread.getStatus() self.assertTrue(status.valid) self.assertEqual(env.event.received, [(('uuid', True), {})]) @permutations([[se.MiscFileReadException], [UnexpectedError]]) def test_from_invalid_to_valid_path(self, exception): with monitor_env() as env: domain = FakeDomain("uuid") monitor.sdCache.domains["uuid"] = domain env.thread.start() # First cycle succeed, but path status fail, emit INVALID event env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult(exception)) del env.event.received[0] # Both domain status and pass status succeed, emit VALID event env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) status = env.thread.getStatus() self.assertTrue(status.valid) self.assertEqual(env.event.received, [(('uuid', True), {})]) def test_keeps_valid(self): with monitor_env() as env: domain = FakeDomain("uuid") monitor.sdCache.domains["uuid"] = domain env.thread.start() # Both domain status and path status succeed and emit VALID event env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) del env.event.received[0] # Both succeed again, no event emitted - domain monitor state did # not change (valid -> valid) env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) status = env.thread.getStatus() self.assertTrue(status.valid) self.assertEqual(env.event.received, []) @permutations([ ("selftest", OSError), ("selftest", UnexpectedError), ("getStats", se.FileStorageDomainStaleNFSHandle), # TODO: Uncomment when test is fixed to not check a type. # ("getStats", se.StorageDomainAccessError("fake-sd-id")), ("getStats", UnexpectedError), ("validateMaster", OSError), ("validateMaster", UnexpectedError), ("getVersion", OSError), ("getVersion", UnexpectedError), ]) def test_from_valid_to_invalid_domain(self, method, exception): with monitor_env() as env: domain = FakeDomain("uuid") monitor.sdCache.domains["uuid"] = domain env.thread.start() # Both domain status and path status succeed and emit VALID event env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) del env.event.received[0] # Domain status fail, emit INVALID event domain.errors[method] = exception env.wait_for_cycle() status = env.thread.getStatus() self.assertFalse(status.valid) # TODO: Should assert we got an exception set by the test instead # of checking a type. self.assertIsInstance(status.error, exception) self.assertEqual(env.event.received, [(('uuid', False), {})]) @permutations([[se.MiscFileReadException], [UnexpectedError]]) def test_from_valid_to_invalid_path(self, exception): with monitor_env() as env: domain = FakeDomain("uuid") monitor.sdCache.domains["uuid"] = domain env.thread.start() # Both domain status and path status succeed and emit VALID event env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) del env.event.received[0] env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult(exception)) status = env.thread.getStatus() self.assertFalse(status.valid) self.assertIsInstance(status.error, exception) self.assertEqual(env.event.received, [(('uuid', False), {})]) def test_acquire_host_id(self): with monitor_env() as env: domain = FakeDomain("uuid") monitor.sdCache.domains["uuid"] = domain env.thread.start() # Both domain status and path status succeed env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) self.assertFalse(domain.acquired) # Acquire host id on the next cycle env.wait_for_cycle() self.assertTrue(domain.acquired) def test_acquire_host_id_after_error(self): with monitor_env() as env: domain = FakeDomain("uuid") domain.errors["selftest"] = OSError monitor.sdCache.domains["uuid"] = domain env.thread.start() # Domain status fail, emit INVALID event env.wait_for_cycle() del domain.errors["selftest"] # Both domain status and path status succeed env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) self.assertFalse(domain.acquired) # Acquire host id on the next cycle env.wait_for_cycle() self.assertTrue(domain.acquired) def test_acquire_host_id_if_lost(self): with monitor_env() as env: domain = FakeDomain("uuid") monitor.sdCache.domains["uuid"] = domain env.thread.start() # Both domain status and path status succeed env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) self.assertFalse(domain.acquired) # Acquire host id on the next cycle env.wait_for_cycle() self.assertTrue(domain.acquired) # Simulate loosing host id - acquire again because status is valid domain.acquired = False env.wait_for_cycle() self.assertTrue(domain.acquired) def test_dont_acquire_host_id_on_iso_domain(self): with monitor_env() as env: domain = FakeDomain("uuid", iso_dir="/path") monitor.sdCache.domains["uuid"] = domain env.thread.start() env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) self.assertFalse(domain.acquired) def test_dont_acquire_host_id_on_error(self): with monitor_env() as env: domain = FakeDomain("uuid") domain.errors["selftest"] = OSError monitor.sdCache.domains["uuid"] = domain env.thread.start() env.wait_for_cycle() env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) self.assertFalse(domain.acquired) @permutations([[se.AcquireHostIdFailure], [UnexpectedError]]) def test_acquire_host_id_retry_after_error(self, exception): with monitor_env() as env: domain = FakeDomain("uuid") domain.errors['acquireHostId'] = exception monitor.sdCache.domains["uuid"] = domain env.thread.start() env.wait_for_cycle() self.assertFalse(domain.acquired) del domain.errors["acquireHostId"] env.checker.complete(domain.getMonitoringPath(), FakeCheckResult()) self.assertFalse(domain.acquired) # Acquire on next cycle env.wait_for_cycle() self.assertTrue(domain.acquired) def test_refresh(self): with monitor_env(refresh=MONITOR_INTERVAL * 1.5) as env: domain = FakeDomain("uuid") monitor.sdCache.domains["uuid"] = domain env.thread.start() # Domain will be removed after the refresh timeout env.wait_for_cycle() self.assertIn(domain.sdUUID, monitor.sdCache.domains) env.wait_for_cycle() # Refresh timeout will expires during next cycle env.wait_for_cycle() self.assertNotIn(domain.sdUUID, monitor.sdCache.domains)